def __init__(self, _id, last_cpu, last_time): self.last_cpu = last_cpu self.last_time = last_time self.client = docker.from_env() if hasattr(self.client, 'api'): self.client = self.client.api self._id = hex(_id)[2:-1].zfill(64)
def docker(): client = docker_client.from_env(assert_hostname=False) try: client.info() except requests_exc.ConnectionError: raise IOError("Cannot connect to docker, is docker running?") return client
def on_get(self, req, resp): """ Send a GET request to get the list of all of the filter containers """ resp.content_type = falcon.MEDIA_TEXT resp.status = falcon.HTTP_200 # connect to docker try: containers = docker.from_env() except Exception as e: # pragma: no cover resp.body = "(False, 'unable to connect to docker because: " + str(e) + "')" return # search for all docker containers and grab ncapture containers container_list = [] try: for c in containers.containers.list(all=True): # TODO: maybe find a way to not have to hard code image name if c.attrs['Config']['Image'] == \ 'cyberreboot/vent-ncapture:master': # the core container is not what we want if 'core' not in c.attrs['Config']['Labels']['vent.groups']: lst = {} lst['id'] = c.attrs['Id'][:12] lst['status'] = c.attrs['State']['Status'] lst['args'] = c.attrs['Args'] container_list.append(lst) except Exception as e: # pragma: no cover resp.body = "(False, 'Failure because: " + str(e) + "')" return resp.body = json.dumps(container_list) return
def test_search(self): client = docker.from_env(timeout=10) res = client.search('busybox') self.assertTrue(len(res) >= 1) base_img = [x for x in res if x['name'] == 'busybox'] self.assertEqual(len(base_img), 1) self.assertIn('description', base_img[0])
def on_get(self, req, resp): """ Send a GET request to get the list of all available network interfaces """ resp.content_type = falcon.MEDIA_TEXT resp.status = falcon.HTTP_200 # connect to docker try: d_client = docker.from_env() except Exception as e: # pragma: no cover resp.body = "(False, 'unable to connect to docker because: " + str(e) + "')" return # start container to get network interfaces nics = '' try: nics = d_client.containers.run('cyberreboot/gonet', network_mode='host', remove=True) resp.body = '(True, ' + str(nics.id) + ')' except Exception as e: # pragma: no cover resp.body = "(False, 'Failure because: " + str(e) + "')" return return
def Services(vent=True): """ Get services that have exposed ports, by default limit to vent containers """ services = [] try: d_client = docker.from_env() if vent: containers = d_client.containers.list(filters={'label':'vent'}) else: containers = d_client.containers.list() for container in containers: if vent: name = container.attrs['Config']['Labels']['vent.name'] else: name = container.name ports = container.attrs['NetworkSettings']['Ports'] p = [] for port in ports: if ports[port]: p.append(ports[port][0]['HostIp']+":"+ports[port][0]['HostPort']) if p: services.append((name, p)) except Exception as e: # pragma: no cover pass return services
def test_exec_run(self): client = docker.from_env(version=TEST_API_VERSION) container = client.containers.run( "alpine", "sh -c 'echo \"hello\" > /test; sleep 60'", detach=True ) self.tmp_containers.append(container.id) assert container.exec_run("cat /test") == b"hello\n"
def test_get(self): client = docker.from_env(version=TEST_API_VERSION) name = helpers.random_name() network_id = client.networks.create(name).id self.tmp_networks.append(network_id) network = client.networks.get(network_id) assert network.name == name
def build_pipeline(self, built_config: Configuration, component_input_config: Dict[str, Any], assembly: Assembly, image_name: str, dist_info: DistInfo, docker_context_dir: Path) -> Pipeline: pipeline = self._default_builder.build_pipeline(built_config, component_input_config, assembly, image_name, dist_info, docker_context_dir) if dist_info.dist_type == DistType.RELEASE: hadoop_version = built_config.components["hadoop"].version build_oozie_stage = BuildOozieStage("distro", DefaultShellCommandExecutor(), hadoop_version) pipeline.inner_stages.insert(0, build_oozie_stage) dependencies = {dependency : built_config.components[dependency] for dependency in assembly.dependencies} # If using Kerberos, add hbase-common-jar-version argument to the Docker build process. if built_config.kerberos: hbase_common_jar_version = component_input_config.get("hbase-common-jar-version", dbd.defaults.HBASE_COMMON_JAR_VERSION) build_args = {"HBASE_COMMON_JAR_VERSION": hbase_common_jar_version} pipeline.final_stage = DefaultPipelineBuilder.get_docker_image_stage(docker.from_env(), image_name, dependencies, docker_context_dir, build_args) return pipeline
def tearDown(self): client = docker.from_env(version=TEST_API_VERSION) for img in self.tmp_imgs: try: client.api.remove_image(img) except docker.errors.APIError: pass for container in self.tmp_containers: try: client.api.remove_container(container, force=True) except docker.errors.APIError: pass for network in self.tmp_networks: try: client.api.remove_network(network) except docker.errors.APIError: pass for volume in self.tmp_volumes: try: client.api.remove_volume(volume) except docker.errors.APIError: pass for folder in self.tmp_folders: shutil.rmtree(folder)
def Containers(vent=True, running=True, exclude_labels=None): """ Get containers that are created, by default limit to vent containers that are running """ containers = [] try: d_client = docker.from_env() if vent: c = d_client.containers.list(all=not running, filters={'label': 'vent'}) else: c = d_client.containers.list(all=not running) for container in c: include = True if exclude_labels: for label in exclude_labels: if 'vent.groups' in container.labels and label in container.labels['vent.groups']: include = False if include: containers.append((container.name, container.status)) except Exception as e: # pragma: no cover logger.error('Docker problem ' + str(e)) return containers
def test_different_user(): current_user_id = os.getuid() if current_user_id != 0: return other_user_id = pwd.getpwnam('nobody').pw_uid cluster = ClickHouseCluster(__file__) node = cluster.add_instance('node') cluster.start() docker_api = docker.from_env().api container = node.get_docker_handle() container.stop() container.start() container.exec_run('chown {} /var/lib/clickhouse'.format(other_user_id), privileged=True) container.exec_run(CLICKHOUSE_START_COMMAND) cluster.shutdown() # cleanup with open(os.path.join(node.path, 'logs/clickhouse-server.err.log')) as log: expected_message = "Effective user of the process \(.*\) does not match the owner of the data \(.*\)\. Run under 'sudo -u .*'\." last_message = log.readlines()[-1].strip() if re.search(expected_message, last_message) is None: pytest.fail('Expected the server to fail with a message "{}", but the last message is "{}"'.format(expected_message, last_message))
def connect(self): """ Connect to a Docker service on which FSMs/SSMs shall be executed. The connection information for this service should be specified with the following environment variables (example for a docker machine installation): export DOCKER_TLS_VERIFY="1" export DOCKER_HOST="tcp://192.168.99.100:2376" export DOCKER_CERT_PATH="/Users/<user>/.docker/machine/machines/default" export DOCKER_MACHINE_NAME="default" Docker machine hint: eval $(docker-machine env default) sets all needed ENV variables. If DOCKER_HOST is not set, the default local Docker socket will be tried. :return: client object """ # lets check if Docker ENV information is set and use local socket as fallback if os.environ.get("DOCKER_HOST") is None: os.environ["DOCKER_HOST"] = "unix://var/run/docker.sock" LOG.warning("ENV variable 'DOCKER_HOST' not set. Using {0} as fallback.".format(os.environ["DOCKER_HOST"])) # lets connect to the Docker instance specified in current ENV # cf.: http://docker-py.readthedocs.io/en/stable/machine/ dc = docker.from_env(assert_hostname=False) # do a call to ensure that we are connected dc.info() LOG.info("Connected to Docker host: {0}".format(dc.base_url)) return dc
def start_msshcopyid_container(containers_dir, image): start_dt = datetime.datetime.now() container_name = 'mssh-copy-id_{0}'.format(str(uuid.uuid4())) container_dir = os.path.join(containers_dir, container_name) volumes = get_container_volumes(container_dir) container_id = start_container(image, name=container_name, detach=True, network=constantstest.DOCKER_NETWORK_NAME, network_alias=container_name, remove=conf.DOCKER_CONTAINERS_TERMINATION == conf.REMOVE, stdin_open=True, tty=False, volumes=volumes, create_volumes_dir=True) client = docker.from_env() ctn = MSSHCopyIdContainer(client.containers.get(container_id)) logger.info(fmt_ctn_log(ctn.name, 'Started the container "%s". (elapsed: %s)'), image, datetime.datetime.now() - start_dt) yield ctn if conf.DOCKER_CONTAINERS_TERMINATION in (conf.STOP, conf.REMOVE): stop_dt = datetime.datetime.now() socket = ctn.attach_socket(params={'stdin': 1, 'stream': 1}) socket.send('exit\n') socket.close() ctn.stop() logger.info(fmt_ctn_log(ctn.name, 'Stopped the container "%s". (elapsed: %s)'), image, datetime.datetime.now() - stop_dt)
def test_logs(self): client = docker.from_env(version=TEST_API_VERSION) container = client.containers.run("alpine", "echo hello world", detach=True) self.tmp_containers.append(container.id) container.wait() assert container.logs() == b"hello world\n"
def test_top(self): client = docker.from_env(version=TEST_API_VERSION) container = client.containers.run("alpine", "sleep 60", detach=True) self.tmp_containers.append(container.id) top = container.top() assert len(top['Processes']) == 1 assert 'sleep 60' in top['Processes'][0]
def main(): state_file = os.path.join(os.path.expanduser("~"), ".dkr", "state.json") state = load_state(state_file) try: docker_client = docker.from_env(assert_hostname=False) script_directory = os.path.dirname(os.path.realpath(__file__)) project_directory = os.path.dirname(script_directory) built_in_modules = load_modules(os.path.join(project_directory, "commands")) user_module_dir = os.path.join(os.path.expanduser("~"), ".dkr", "commands") user_modules = load_modules(user_module_dir) modules = {} modules.update(built_in_modules) modules.update(user_modules) arg_parser = argparse.ArgumentParser(description="Extensible Docker CLI Client") subparsers = arg_parser.add_subparsers(title="Commands", metavar="COMMAND") for name, module in modules.items(): if hasattr(module, 'import_command'): subparser = subparsers.add_parser(name, help=module.help_summary(name) if hasattr(module, 'help_summary') else "Does something wonderful!") module.import_command(docker_client, subparser, state) parsed_args = arg_parser.parse_args() if 'func' in parsed_args: parsed_args.func(docker_client, parsed_args, state) else: print("No valid command specified. `{} -h` for help.".format(sys.argv[0])) finally: save_state(state, state_file)
def __init__(self): self.logger = logging.getLogger(__name__) self.docker = docker.from_env() if len(self.docker.images(name=self.image_name)) == 0: self._fetch_image() host_config = self.docker.create_host_config( binds=self.volumes_bindings, port_bindings=self.port_bindings ) try: # following avoids conflict between different builds container_name_prefixed = '{}_{}'.format(os.path.basename(os.path.realpath('{}/../..'.format(__file__))), self.container_name) self.container_id = self.docker.create_container(self.image_name, name=container_name_prefixed, ports=self.ports, command=self.command, environment=self.env_vars, volumes=self.volumes, host_config=host_config).get('Id') self.logger.info("docker id is {}".format(self.container_id)) self.logger.info("starting the temporary docker for image {}".format(self.image_name)) self.docker.start(self.container_id) self.ip_addr = self.docker.inspect_container(self.container_id).get('NetworkSettings', {}).get('IPAddress') if not self.ip_addr: self.logger.error("temporary docker {} not started".format(self.container_id)) assert False self.logger.info("IP addr is {}".format(self.ip_addr)) self.wait_until_available() except APIError as e: pytest.exit( "error during setup of docker container {}, aborting. Details:\n{}".format(container_name_prefixed, str(e)))
def active_directory(tmpdir): environment = { 'SAMBA_DOMAIN': 'openforce', 'SAMBA_HOST_NAME': 'dc', 'SAMBA_ADMINPASS': '******', 'SAMBA_KRBTGTPASS': '******', 'SAMBA_REALM': 'OPENFORCE.ORG', } ports = { '22': 2222, '53': 5353, '88': 88, '135': 135, '139': 139, '389': 389, '445': 445, '464': 464, '636': 636, '1024': 1024, '3268': 3268, '3269': 3269 } client = docker.from_env() dc = None if not client.containers.list(all, filters={'name': 'dc'}): dc = client.containers.run('xnandersson/samba-ad-dc', command='dcpromo.py', privileged=True, ports=ports, name='dc', environment=environment, detach=True) time.sleep(10) else: dc = client.containers.get('dc') dc.start() time.sleep(2) yield dc.kill()
def test_create(self): client = docker.from_env(version=TEST_API_VERSION) name = helpers.random_name() network = client.networks.create(name, labels={'foo': 'bar'}) self.tmp_networks.append(network.id) assert network.name == name assert network.attrs['Labels']['foo'] == "bar"
def task_cleanup(e): """ Since files written by docker containers are owned by root, we can't clean them up in the worker process since that typically doesn't run as root. So, we run a lightweight container to make the temp dir cleanable. """ from .executor import DATA_VOLUME if e.info['task']['mode'] == 'docker' and '_tempdir' in e.info['kwargs']: tmpdir = e.info['kwargs']['_tempdir'] client = docker.from_env(version='auto') config = { 'tty': True, 'volumes': { tmpdir: { 'bind': DATA_VOLUME, 'mode': 'rw' } }, 'detach': False, 'remove': True } args = ['chmod', '-R', 'a+rw', DATA_VOLUME] try: client.containers.run('busybox:latest', args, **config) except DockerException as dex: logger.error('Error setting perms on docker tempdir %s.' % tmpdir) logger.exception(dex) raise
def start_sshd_container(containers_dir): start_dt = datetime.datetime.now() docker_image = constantstest.DOCKER_SSHD_IMAGE container_name = 'sshd_{0}'.format(str(uuid.uuid4())) container_dir = os.path.join(containers_dir, container_name) volumes = get_container_volumes(container_dir) container_id = start_container(docker_image, name=container_name, detach=True, network=constantstest.DOCKER_NETWORK_NAME, network_alias=container_name, remove=conf.DOCKER_CONTAINERS_TERMINATION == conf.REMOVE, stdin_open=True, tty=False, volumes=volumes, create_volumes_dir=True) client = docker.from_env() ctn = Container(client.containers.get(container_id)) logger.info(fmt_ctn_log(ctn.name, 'Started the container "%s". (elapsed: %s)'), docker_image, datetime.datetime.now() - start_dt) yield ctn if conf.DOCKER_CONTAINERS_TERMINATION in (conf.STOP, conf.REMOVE): stop_dt = datetime.datetime.now() ctn.stop() logger.info(fmt_ctn_log(ctn.name, 'Stopped the container "%s". (elapsed: %s)'), docker_image, datetime.datetime.now() - stop_dt)
def test_clone(self): """ test creating a container given an image. """ count = 3 host1 = docker.from_env() self.assertTrue(host1) pool = api.Pool(CONNECTION, "test") self.assertIsNotNone(pool) for item in range(count): name = pool.new_name_get(TEMPLATE, item) pool.destroy(name) names = [item for item in pool.conn.containers.list()] for item in range(count): name = pool.new_name_get(TEMPLATE, item) if name not in names: logging.debug("cloning %s to %s", TEMPLATE, name) pool.clone(TEMPLATE, name) pool.start(name) for item in range(count): name = TEMPLATE + ".%d" % item pool.destroy(name)
def system_commands(self, action): """ Perform system commands """ if action == "reset": okay = npyscreen.notify_ok_cancel( "This factory reset will remove ALL of Vent's user data, " "containers, and images. Are you sure?", title="Confirm system command") if okay: d_cli = docker.from_env() # remove containers list = d_cli.containers.list(filters={'label':'vent'}, all=True) for c in list: c.remove(force=True) # remove images list = d_cli.images.list(filters={'label':'vent'}, all=True) for i in list: d_cli.images.remove(image=i.id, force=True) # remove .vent folder try: shutil.rmtree(os.path.join(os.path.expanduser('~'),'.vent')) except Exception as e: # pragma: no cover npyscreen.notify_confirm("Error deleting Vent data: "+repr(e)) else: npyscreen.notify_confirm("Vent reset complete. " "Press OK to exit Vent Manager console.") self.exit() pass elif action == "upgrade": # !! TODO pass return
def execute(self, code=None, files=None, command=None, produces=None, docker_options=None): files = files or {} docker_options = docker_options or {} _tempdir = docker_options.get('tempdir') _mem_limit = docker_options.get('mem_limit') _cpu_period = docker_options.get('cpu_period') _cpu_quota = docker_options.get('cpu_quota') _timeout = docker_options.get('timeout') client = docker.from_env() with tempfile.TemporaryDirectory(dir=_tempdir) as tempdir: temp_path = Path(tempdir) self._write_files(temp_path, code, files) docker_command = self._get_docker_command( tempdir, code, command, files) container = client.containers.run( self.docker_image, docker_command, detach=True, volumes={ str(temp_path): {'bind': '/app'} }, mem_limit=_mem_limit, cpu_period=_cpu_period, cpu_quota=_cpu_quota, working_dir='/app') result = { 'execution_error': None } try: exit_status = container.wait(timeout=_timeout) result.update({ 'successful': exit_status == 0 }) if produces: result['files'] = self._get_produced_files( temp_path, produces) except ReadTimeout: result.update({ 'successful': False, 'execution_error': 'timeout' }) from docker.errors import APIError if container.status in KILLABLE_STATUS: try: container.kill() except APIError: pass result.update({ 'stdout': container.logs(stdout=True, stderr=False), 'stderr': container.logs(stdout=False, stderr=True), }) container.remove() return result
def __init__(self, image="phusion/baseimage:latest", verbose=True): self.image = image self.client = docker.from_env() self.base = self.client.images.pull(self.image) self.container = self.client.containers.run(self.base, "/sbin/my_init", detach=True) self.verbose = verbose self.docker_id = str(self.container.id[:12])
def get_docker_api(self, docker_api): if docker_api == 'local': # commect to local docker api return docker.from_env() else: # connect to remote docker pai eg. tcp://127.0.0.1:1234 return docker.DockerClient(base_url=docker_api)
def compile( submission ): ''' Compile the target submission in work_dir Return status , info if staus eq 'Compile Error' or 'Judger Error' then info would show reasons and in any onther cases, the info would be None ''' client = docker.from_env() try: s = client.containers.run( image = settings.docker_repo_arguments.format( repo_lang = submission.language.value.image ), network_disabled = True, volumes = { submission.work_dir : {'bind': '/opt' , 'mode':'rw' } }, working_dir = '/opt', mem_limit = settings.COMPILE_MEMORY, auto_remove = True, tty = True, detach = True) status, info = s.exec_run( cmd = 'timeout ' + str(settings.COMPILE_TIMEOUT) + ' ' + submission.language.value.compile_command.format( sourcefile = submission.sourcefile)) status = int( status ) info = info.decode( 'utf-8' ) finally: if 's' in dir(): s.remove( force = True ) # No matter how this container work, we should remove this container force finally if status == 0: return 'Success' , None elif status is 124: info = 'Compile time out' elif len( info ) == 0: info = 'You wanna hack me? exit code is ' + str( status ) return Judge_result.CE , info[:min( len(info) , settings.max_compile_error_length )]
def Docker(): """ Get Docker setup information """ docker_info = {'server':{}, 'env':'', 'type':'', 'os':''} # get docker server version try: d_client = docker.from_env() docker_info['server'] = d_client.version() except Exception as e: # pragma: no cover pass # get operating system system = System() docker_info['os'] = system # check if native or using docker-machine if 'DOCKER_MACHINE_NAME' in os.environ: # using docker-machine docker_info['env'] = os.environ['DOCKER_MACHINE_NAME'] docker_info['type'] = 'docker-machine' elif 'DOCKER_HOST' in os.environ: # not native docker_info['env'] = os.environ['DOCKER_HOST'] docker_info['type'] = 'remote' else: # using "local" server docker_info['type'] = 'native' return docker_info
def waiting_container(job_id): """ This method wait for the end of the execution of the container in another thread (to not block the main application) and update job status accordingly) """ from core.core import core from core.model import Job job = Job.from_id(job_id) container_name = DOCKER_CONFIG["job_name"].format("{}_{}".format(job.pipeline_id, job.id)) try: container = docker.from_env().containers.get(container_name) except Exception as ex: pass # wait until the container stops container.wait() # refresh job information job = Job.from_id(job_id, 1) if job.status == 'running': # The container stop auto when its job is done, so have to finalyse it job.progress_value = 1.0 job.save() core.jobs.finalize(job_id)
import argparse import docker from tensorforce.execution import ParallelRunner from tensorforce.execution import Runner #from pommerman.runner import ExperimentRunner from tensorforce.environments.openai_gym import OpenAIGym import gym from pommerman import helpers, make from pommerman.agents import TensorForceAgent import timeit import pickle import matplotlib.pyplot as plt CLIENT = docker.from_env() save_name = 'ppo' def save_obj(obj): with open('./saved_models/' + save_name + '.pkl', 'wb') as f: pickle.dump(obj, f, pickle.HIGHEST_PROTOCOL) def load_obj(): try: with open('./saved_models/' + save_name + '.pkl', 'rb') as f: return pickle.load(f) except: return []
def __init__(self, image_name, validator_url, port=3020): self.__client = docker.from_env() self.container = self.__launch(self.__client, image_name, port) super().__init__(validator_url)
def __init__(self, manifest, *args, **kwargs): self.path_dirs = PathDirs(**kwargs) self.manifest = manifest self.d_client = docker.from_env() self.logger = Logger(__name__)
--config=PommeFFACompetition-v0 """ import atexit import functools import os import argparse import docker from tensorforce.execution import Runner from tensorforce.contrib.openai_gym import OpenAIGym import gym from .. import helpers, make from ..agents import TensorForceAgent client = docker.from_env() def clean_up_agents(agents): """Stops all agents""" return [agent.shutdown() for agent in agents] class WrappedEnv(OpenAIGym): def __init__(self, gym, visualize=False): self.gym = gym self.visualize = visualize def execute(self, actions): if self.visualize: self.gym.render()
def tearDownClass(cls) -> None: if not cls.BBLFSH_SERVER_EXISTED: client = docker.from_env(version="auto") client.containers.get("bblfshd").remove(force=True) client.api.close()
def get_docker_info(): client = docker.from_env() docker_info = client.info() docker_version = docker_info.get('ServerVersion') # ServerVersion storage_driver = docker_info.get('Driver') # 存储驱动 containers = docker_info.get('Containers') # 容器数量 containers_running = docker_info.get('ContainersRunning') # 运行容器数量 containers_paused = docker_info.get('ContainersPaused') # 暂停运行的容器数量 containers_stopped = docker_info.get('ContainersStopped') # 停止运行的容器数量 containers_images = docker_info.get('Images') # 容器镜像数量 docker_info_dict = { 'storage_driver': storage_driver, 'containers': containers, 'containers_running': containers_running, 'containers_paused': containers_paused, 'containers_stopped': containers_stopped, 'containers_images': containers_images, 'docker_version': docker_version, 'containers_list': [], } containers_id_str = subprocess.Popen( "docker ps -a | grep -v 'COMMAND' | awk ' { print $1 } '", stdout=subprocess.PIPE, shell=True).stdout.read().decode() if containers_id_str: containers_id_list = containers_id_str.split('\n') for containers_id in containers_id_list: if containers_id: container = client.containers.get(containers_id) if container.attrs.get('Id') not in docker_info_dict: docker_info_dict.get('containers_list').append({ container.attrs.get('Id'): { 'Image': container.attrs.get('Config').get('Image'), 'Env': container.attrs.get('Config').get('Env'), 'WorkingDir': container.attrs.get('Config').get('WorkingDir'), 'Platform': container.attrs.get('Platform'), 'Created': container.attrs.get('Created'), 'State': { 'StartedAt': container.attrs.get('State').get('StartedAt'), 'Status': container.attrs.get('State').get('Status') }, 'Network': container.attrs.get('NetworkSettings').get( 'Networks').keys(), 'Ports': container.attrs.get('NetworkSettings').get( 'Ports'), 'Name': container.attrs.get('Name'), 'mounts': { mountskeys.get('Destination'): mountskeys.get('Source') for mountskeys in container.attrs.get('Mounts') } } }) return docker_info_dict
def compileSrc(): client = docker.from_env() containerObj = client.containers.run(image = "test", remove = False, detach = True) time.sleep(1) containerObj.stop() writeLogsFile(containerObj)