def __init__(self, _exit, verbose): # setup_client try: # setup client depending if running on linux or using boot2docker (osx/win) if sys.platform == 'linux': self.client = docker_py.Client(version='auto') else: # get b2d ip self.ip = str(sh.boot2docker.ip()).strip() try: # try secure connection first kw = kwargs_from_env(assert_hostname=False) self.client = docker_py.Client(version='auto', **kw) except docker_py.errors.DockerException as e: # shit - some weird boot2docker, python, docker-py, requests, and ssl error # https://github.com/docker/docker-py/issues/465 if verbose: log.debug(e) log.warn("Cannot connect securely to Docker, trying insecurely") kw = kwargs_from_env(assert_hostname=False) if 'tls' in kw: kw['tls'].verify = False self.client = docker_py.Client(version='auto', **kw) except Exception as e: if verbose: log.error("Could not connect to Docker - try running 'docker info' first") if sys.platform != 'linux': log.error("Please ensure you've run 'boot2docker up' and 'boot2docker shellinit' first and have added the ENV VARs it suggests") if _exit: raise e
def docker_client(args): """ Attempts to create a docker client. - args: The arguments parsed on the command line. - returns: a docker-py client """ if _platform == 'linux' or _platform == 'linux2': # linux if "docker_url" in args: return Client(base_url=args.docker_url) else: # TODO: test to see if this does the right thing by default. return Client(**kwargs_from_env()) elif _platform == 'darwin': # OS X - Assume boot2docker, and pull from that environment. kwargs = kwargs_from_env() if not args.strict_docker_tls: kwargs['tls'].assert_hostname = False return Client(**kwargs) elif _platform == 'win32' or _platform == 'cygwin': # Windows. logging.fatal("Sorry, windows is not currently supported!") sys.exit(2)
def docker_client(args): """ Attempts to create a docker client. - args: The arguments parsed on the command line. - returns: a docker-py client """ if _platform == 'linux' or _platform == 'linux2': # linux if "docker_url" in args: return Client( base_url=args.docker_url, timeout=args.timeout, version='auto') else: # TODO: test to see if this does the right thing by default. return Client( version='auto', timeout=args.timeout, **kwargs_from_env()) elif _platform == 'darwin': # OS X - Assume boot2docker, and pull from that environment. kwargs = kwargs_from_env() if len(kwargs) == 0: logging.error('Could not correctly pull in docker environment. ' 'Try running: eval "$(docker-machine env default)"') sys.exit(2) if not args.strict_docker_tls: kwargs['tls'].assert_hostname = False return Client(version='auto', timeout=args.timeout, **kwargs) elif _platform == 'win32' or _platform == 'cygwin': # Windows. logging.fatal("Sorry, windows is not currently supported!") sys.exit(2)
def get_docker_client(): """ Universal method to use docker.client() """ try: return docker.AutoVersionClient(**kwargs_from_env()) except docker.errors.DockerException: return docker.Client(**kwargs_from_env())
def test_auto_client(self): client = docker.AutoVersionClient(**kwargs_from_env()) client_version = client._version api_version = client.version(api_version=False)['ApiVersion'] self.assertEqual(client_version, api_version) api_version_2 = client.version()['ApiVersion'] self.assertEqual(client_version, api_version_2) client.close() with self.assertRaises(docker.errors.DockerException): docker.AutoVersionClient(version='1.11', **kwargs_from_env())
def setup_machine_client(self, verbose): try: # try secure connection first kw = kwargs_from_env(assert_hostname=False) return docker_py.Client(version='auto', **kw) except docker_py.errors.DockerException as e: # shit - some weird boot2docker, python, docker-py, requests, and ssl error # https://github.com/docker/docker-py/issues/465 if verbose: log.debug(e) log.warn("Cannot connect securely to Docker, trying insecurely") kw = kwargs_from_env(assert_hostname=False) if 'tls' in kw: kw['tls'].verify = False return docker_py.Client(version='auto', **kw)
def build_buildcontainer_image(base_path): assert_initialized(base_path) # To ensure version compatibility, we have to generate the kwargs ourselves client_kwargs = kwargs_from_env(assert_hostname=False) client = docker.AutoVersionClient(**client_kwargs) with make_temp_dir() as temp_dir: logger.info('Building Docker Engine context...') tarball_path = os.path.join(temp_dir, 'context.tar') tarball_file = open(tarball_path, 'wb') tarball = tarfile.TarFile(fileobj=tarball_file, mode='w') container_dir = os.path.normpath(os.path.join(base_path, 'ansible')) try: tarball.add(container_dir, arcname='ansible') except OSError: raise AnsibleContainerNotInitializedException() jinja_render_to_temp('ansible-dockerfile.j2', temp_dir, 'Dockerfile') tarball.add(os.path.join(temp_dir, 'Dockerfile'), arcname='Dockerfile') jinja_render_to_temp( 'hosts.j2', temp_dir, 'hosts', hosts=extract_hosts_from_docker_compose(base_path)) tarball.add(os.path.join(temp_dir, 'hosts'), arcname='hosts') tarball.close() tarball_file = open(tarball_path, 'rb') logger.info('Starting Docker build of Ansible Container image...') return [ streamline for streamline in client.build(fileobj=tarball_file, rm=True, custom_context=True, pull=True, forcerm=True, tag='ansible-container-builder') ]
def docker_client(environment, version=None, tls_config=None, host=None, tls_version=None): """ Returns a docker-py client configured using environment variables according to the same logic as the official Docker client. """ try: kwargs = kwargs_from_env(environment=environment, ssl_version=tls_version) except TLSParameterError: raise UserError( "TLS configuration is invalid - make sure your DOCKER_TLS_VERIFY " "and DOCKER_CERT_PATH are set correctly.\n" "You might need to run `eval \"$(docker-machine env default)\"`") if host: kwargs['base_url'] = host if tls_config: kwargs['tls'] = tls_config if version: kwargs['version'] = version timeout = environment.get('COMPOSE_HTTP_TIMEOUT') if timeout: kwargs['timeout'] = int(timeout) else: kwargs['timeout'] = HTTP_TIMEOUT kwargs['user_agent'] = generate_user_agent() return APIClient(**kwargs)
def run(self): while (True): ltime, container_id, max_time = self._input_queue.get() time_diff = ltime - time.time() if time_diff > 0: # we still have to wait self._input_queue.put((ltime, container_id, max_time)) time.sleep(min(time_diff, 5)) # wait maximum for 5 seconds else: try: usage = float(get_container_cgroup("cpuacct", container_id).get_stats()['usage']) except: continue # container has closed self.logger.debug("Current time usage: %f. Max: %f.", (usage / (10 ** 9)), max_time) if (usage / (10 ** 9)) < max_time: # retry minimum_remaining_time = math.ceil((max_time - (usage / (10 ** 9))) / self._cpu_count) self.logger.debug("Minimum wait: %f.", minimum_remaining_time) self._input_queue.put((time.time() + minimum_remaining_time, container_id, max_time)) else: # kill it (with fire!) self.logger.info("Killing container %s due to timeout", container_id) self._container_errors.add(container_id) try: docker_connection = docker.Client(**kwargs_from_env()) docker_connection.kill(container_id) except: pass
def main(): logging.basicConfig(level=logging.INFO, format="%(message)s", stream=sys.stdout) args = get_args() client = docker.APIClient(version='auto', timeout=args.timeout, **kwargs_from_env()) exclude_container_labels = format_exclude_labels( args.exclude_container_label) if args.max_container_age: cleanup_containers( client, args.max_container_age, args.dry_run, exclude_container_labels, ) if args.max_image_age: exclude_set = build_exclude_set(args.exclude_image, args.exclude_image_file) cleanup_images(client, args.max_image_age, args.dry_run, exclude_set) if args.dangling_volumes: cleanup_volumes(client, args.dry_run)
def world_of_docker(): """Pulls and starts a container from a random Docker image in the top 100.""" with open('repos.json') as data_file: repos = json.load(data_file) client = Client(**kwargs_from_env()) random_repo = random.choice(repos)['name'] click.echo('Hmmmmmm.... how about %s? Everybody likes %s!' % (random_repo, random_repo)) for line in client.pull(random_repo, stream=True): click.echo(json.loads(line)['status']) click.echo('Now let\'s just start up a container here...') container = client.create_container(image=random_repo) client.start(container=container.get('Id')) container_name = client.inspect_container( container['Id'])['Name'].strip('/') click.echo('Up and running! Enjoy your new %s container, %s' % (random_repo, container_name))
def test_dump_file(tmpdir, docker_client): dump_file = tmpdir.join('dump.txt') tmp = tmpdir.join('shipwright-sample') path = str(tmp) source = pkg_resources.resource_filename( __name__, 'examples/shipwright-sample', ) create_repo(path, source) client_cfg = docker_utils.kwargs_from_env() cli = docker_client try: args = get_defaults() args['--dump-file'] = str(dump_file) shipw_cli.run( path=path, client_cfg=client_cfg, arguments=args, environ={}, ) assert ' : FROM busybox' in dump_file.read() finally: old_images = (cli.images(name='shipwright/service1', quiet=True) + cli.images(name='shipwright/shared', quiet=True) + cli.images(name='shipwright/base', quiet=True)) for image in old_images: cli.remove_image(image, force=True)
def push_image(self): """Push docker image to registry""" client = docker.APIClient(version="auto", **kwargs_from_env()) # Build a progress setup for each layer, and only emit per-layer # info every 1.5s layers = {} last_emit_time = time.time() for line in client.push(self.output_image_spec, stream=True): progress = json.loads(line.decode("utf-8")) if "error" in progress: self.log.error(progress["error"], extra=dict(phase="failed")) raise docker.errors.ImageLoadError(progress["error"]) if "id" not in progress: continue if "progressDetail" in progress and progress["progressDetail"]: layers[progress["id"]] = progress["progressDetail"] else: layers[progress["id"]] = progress["status"] if time.time() - last_emit_time > 1.5: self.log.info("Pushing image\n", extra=dict(progress=layers, phase="pushing")) last_emit_time = time.time() self.log.info( "Successfully pushed {}".format(self.output_image_spec), extra=dict(phase="pushing"), )
def test_sample(tmpdir, capsys): path = str(tmpdir.join('shipwright-sample')) source = pkg_resources.resource_filename( __name__, 'examples/shipwright-sample', ) repo = create_repo(path, source) tag = repo.head.ref.commit.hexsha[:12] client_cfg = docker_utils.kwargs_from_env() args = get_defaults() args['images'] = True shipw_cli.run( path=path, client_cfg=client_cfg, arguments=args, environ={}, ) out, err = capsys.readouterr() images = {'base', 'shared', 'service1'} tmpl = 'shipwright/{img}:{tag}' expected = {tmpl.format(img=i, tag=tag) for i in images} assert {l for l in out.split('\n') if l} == expected
def __init__(self, bs, worker_config, args): Worker.__init__(self, bs, worker_config, args) self.client = docker.Client( version=os.environ.get('DOCKER_VERSION'), **kwargs_from_env(assert_hostname=False) ) log.info('Connecting to docker daemon ...') try: image = args.image if ':' in image: self.image, self.tag = image.split(':', 1) else: self.image, self.tag = image, None images = self.client.images(self.image) except ConnectionError as err: raise errors.BinstarError( "Docker client could not connect to daemon (is docker installed?)\n" "You may need to set your DOCKER_HOST environment variable") if not images: raise errors.BinstarError( "You do not have the docker image '{image}'\n" "You may need to run:\n\n\tdocker pull {image}s\n".format(image=args.image)) if self.args.allow_user_images: log.warn("Allowing users to specify docker images")
def client(self): """single global client instance""" cls = self.__class__ if cls._client is None: if self.use_docker_client_env: kwargs = kwargs_from_env( assert_hostname=self.tls_assert_hostname ) client = docker.Client(version='auto', **kwargs) else: if self.tls: tls_config = True elif self.tls_verify or self.tls_ca or self.tls_client: tls_config = docker.tls.TLSConfig( client_cert=self.tls_client, ca_cert=self.tls_ca, verify=self.tls_verify, assert_hostname = self.tls_assert_hostname) else: tls_config = None docker_host = os.environ.get('DOCKER_HOST', 'unix://var/run/docker.sock') client = docker.Client(base_url=docker_host, tls=tls_config, version='auto') cls._client = client return cls._client
async def postgres(loop): tag = 'latest' image = 'postgres' host = '127.0.0.1' timeout = 60 unused_tcp_port = get_free_port() client = DockerClient(version='auto', **kwargs_from_env()) client.images.pull(image, tag=tag) print('Stating %s:%s on %s:%s' % (image, tag, host, unused_tcp_port)) cont = client.containers.run('%s:%s' % (image, tag), detach=True, ports={'5432/tcp': ('0.0.0.0', unused_tcp_port)}) try: start_time = time.time() conn = None while conn is None: if start_time + timeout < time.time(): raise Exception("Initialization timeout, failed to " "initialize postgresql container") try: conn = await asyncpg.connect( 'postgresql://postgres@%s:%s/postgres' '' % (host, unused_tcp_port), loop=loop) except Exception as e: time.sleep(.1) await conn.close() yield (host, unused_tcp_port) finally: cont.kill() cont.remove()
def download_docker_image(docker_image, target_file, cache=None): try: from docker.client import Client from docker.utils import kwargs_from_env kwargs = kwargs_from_env() kwargs['tls'].assert_hostname = False docker_cli = Client(**kwargs) image = docker_cli.get_image(docker_image) image_tar = open(target_file,'w') image_tar.write(image.data) image_tar.close() except Exception as e: if cache is not None: cached_file = os.path.join(cache, docker_image.lower().replace('/','-').replace(':','-') + '.tgz') if os.path.isfile(cached_file): print 'using cached version of', docker_image urllib.urlretrieve(cached_file, target_file) return print >> sys.stderr, docker_image, 'not found in cache', cache sys.exit(1) if isinstance(e, KeyError): print >> sys.stderr, 'docker not configured on this machine (or environment variables are not properly set)' else: print >> sys.stderr, docker_image, 'not found on local machine' print >> sys.stderr, 'you must either pull the image, or download it and use the --docker-cache option' sys.exit(1)
def test_exact(tmpdir, docker_client): path = str(tmpdir.join('shipwright-sample')) source = pkg_resources.resource_filename( __name__, 'examples/shipwright-sample', ) create_repo(path, source) client_cfg = docker_utils.kwargs_from_env() cli = docker_client args = get_defaults() args['--exact'] = [ 'shipwright/base', ] try: shipw_cli.run( path=path, client_cfg=client_cfg, arguments=args, environ={}, ) base, = (cli.images(name='shipwright/service1') + cli.images(name='shipwright/shared') + cli.images(name='shipwright/base')) assert 'shipwright/base:master' in base['RepoTags'] assert 'shipwright/base:latest' in base['RepoTags'] finally: old_images = (cli.images(name='shipwright/base', quiet=True)) for image in old_images: cli.remove_image(image, force=True)
def main(): args = parse_args( sys.argv[1:] ) kwargs = kwargs_from_env() # Connect to docker if len( kwargs.keys() ) == 0: logger.warning( 'Unable to discover Docker settings through env' ) logger.info( 'Using {}'.format( args.docker ) ) kwargs['base_url'] = args.docker # Connect to ECTD etcd_client = etcd.Client( host=args.etcd_host, port=args.etcd_port ) etcd_folder = os.path.join( args.prefix, args.domain ) logger.debug( 'Announcing to {}'.format( etcd_folder ) ) # Find the matching container docker_client = Client(**kwargs) try: containers = docker_client.containers() logger.error( containers ) except Exception as e: logger.error( e ) sys.exit( 'FAILURE - Unable to connect Docker. Is it running?' ) # Find the matching container matching = find_matching_container( containers, args ) # Main health checking loop while True: announce_services( matching.items(), etcd_folder, etcd_client, args.timeout, args.ttl, args.vulcand )
def __init__(self): kwargs = kwargs_from_env(assert_hostname=False) kwargs.update({'version': 'auto'}) self.client = Client(**kwargs) try: self.client.info() # mount the /Volumes folder under mac, please refer to # https://github.com/bpeng2000/SOS/wiki/SoS-Docker-guide # for details. self.has_volumes = False if platform.system() == 'Darwin': try: # this command log in to the docker machine, check if /Volumes has been mounted, # and try to mount it if possible. This requires users to configure subprocess.call( """docker-machine ssh "{}" 'mount | grep /Volumes || {{ echo "mounting /Volumes"; sudo mount -t vboxsf Volumes /Volumes; }}' """ .format(os.environ['DOCKER_MACHINE_NAME']), shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) env.logger.trace( 'Sucessfully mount /Volumes to virtual machine') self.has_volumes = True except Exception as e: env.logger.trace( 'Failed to mount /Volumes to virtual machine: {}'. format(e)) except Exception as e: env.logger.debug('Docker client init fail: {}'.format(e)) self.client = None
def docker_client(environment, version=None, context=None, tls_version=None): """ Returns a docker-py client configured using environment variables according to the same logic as the official Docker client. """ try: kwargs = kwargs_from_env(environment=environment, ssl_version=tls_version) except TLSParameterError: raise UserError( "TLS configuration is invalid - make sure your DOCKER_TLS_VERIFY " "and DOCKER_CERT_PATH are set correctly.\n" "You might need to run `eval \"$(docker-machine env default)\"`") if not context: # check env for DOCKER_HOST and certs path host = kwargs.get("base_url", None) tls = kwargs.get("tls", None) verify = False if not tls else tls.verify if host: context = Context("compose", host=host, tls=verify) else: context = ContextAPI.get_current_context() if tls: context.set_endpoint("docker", host=host, tls_cfg=tls, skip_tls_verify=not verify) if not context.is_docker_host(): raise UserError( "The platform targeted with the current context is not supported.\n" "Make sure the context in use targets a Docker Engine.\n") kwargs['base_url'] = context.Host if context.TLSConfig: kwargs['tls'] = context.TLSConfig if version: kwargs['version'] = version timeout = environment.get('COMPOSE_HTTP_TIMEOUT') if timeout: kwargs['timeout'] = int(timeout) else: kwargs['timeout'] = HTTP_TIMEOUT kwargs['user_agent'] = generate_user_agent() # Workaround for # https://pyinstaller.readthedocs.io/en/v3.3.1/runtime-information.html#ld-library-path-libpath-considerations if 'LD_LIBRARY_PATH_ORIG' in environment: kwargs['credstore_env'] = { 'LD_LIBRARY_PATH': environment.get('LD_LIBRARY_PATH_ORIG'), } use_paramiko_ssh = int(environment.get('COMPOSE_PARAMIKO_SSH', 0)) client = APIClient(use_ssh_client=not use_paramiko_ssh, **kwargs) client._original_base_url = kwargs.get('base_url') return client
def create_local_test_data(private=False): appyaml = { 'appname': 'blueberry', 'entrypoints': { 'web': { 'cmd': 'python app.py', 'ports': ['5000/tcp'], }, 'daemon': { 'cmd': 'python daemon.py', }, 'service': { 'cmd': 'python service.py' }, }, 'build': 'pip install -r ./requirements.txt', } app = App.get_or_create('blueberry', 'http://git.hunantv.com/tonic/blueberry.git', 'token') version = app.add_version('abe23812aeb50a17a2509c02a28423462161d306') appconfig = version.appconfig appconfig.update(**appyaml) appconfig.save() group = Group.create('group', 'group') pod = Pod.create('pod', 'pod') pod.assigned_to_group(group) c = docker.Client(**kwargs_from_env(assert_hostname=False)) r = c.info() host = Host.create(pod, '192.168.59.103:2376', r['Name'], r['ID'], r['NCPU'], r['MemTotal']) if private: host.assigned_to_group(group) return app, version, group, pod, host
def test_kwargs_from_env_empty(self): os.environ.update(DOCKER_HOST='', DOCKER_CERT_PATH='') os.environ.pop('DOCKER_TLS_VERIFY', None) kwargs = kwargs_from_env() self.assertEqual(None, kwargs.get('base_url')) self.assertEqual(None, kwargs.get('tls'))
def have_docker(): if os.environ.get('NO_DOCKER_TESTS'): return False try: import docker from docker.utils import kwargs_from_env except: warnings.warn("Skip Docker Tests: dockerpy is not installed") return False client = docker.Client( version=os.environ.get('DOCKER_VERSION'), **kwargs_from_env(assert_hostname=False) ) try: images = client.images('binstar/linux-64') except docker.errors.NotFound as err: warnings.warn("Skip Docker Tests: {}".format(err)) return False except requests.ConnectionError: warnings.warn("Skip Docker Tests: image binstar/linux-64 is not pulled") return False return True
def __init__( self, docker_host='unix://var/run/docker.sock', version='auto', timeout=30, max_workers=64, assert_hostname=False, ): #kwargs = kwargs_from_env(assert_hostname=False) kwargs = kwargs_from_env(assert_hostname=assert_hostname) # environment variable DOCKER_HOST takes precedence kwargs.setdefault('base_url', docker_host) blocking_docker_client = docker.Client(version=version, timeout=timeout, **kwargs) executor = ThreadPoolExecutor(max_workers=max_workers) async_docker_client = AsyncDockerClient(blocking_docker_client, executor) self.docker_client = async_docker_client self.port = 0
def __init__(self, bs, worker_config, args): Worker.__init__(self, bs, worker_config, args) self.client = docker.Client(version=os.environ.get('DOCKER_VERSION'), **kwargs_from_env(assert_hostname=False)) log.info('Connecting to docker daemon ...') try: image = args.image if ':' in image: self.image, self.tag = image.split(':', 1) else: self.image, self.tag = image, None images = self.client.images(self.image) except ConnectionError as err: raise errors.BinstarError( "Docker client could not connect to daemon (is docker installed?)\n" "You may need to set your DOCKER_HOST environment variable") if not images: raise errors.BinstarError( "You do not have the docker image '{image}'\n" "You may need to run:\n\n\tdocker pull {image}s\n".format( image=args.image)) if self.args.allow_user_images: log.warn("Allowing users to specify docker images")
def get_client(self): if not self._client: # To ensure version compatibility, we have to generate the kwargs ourselves client_kwargs = kwargs_from_env(assert_hostname=False) self._client = docker.AutoVersionClient(**client_kwargs) self.api_version = self._client.version()['ApiVersion'] return self._client
def have_docker(): if os.environ.get('NO_DOCKER_TESTS'): return False try: import docker from docker.utils import kwargs_from_env except: warnings.warn("Skip Docker Tests: dockerpy is not installed") return False client = docker.Client(version=os.environ.get('DOCKER_VERSION'), **kwargs_from_env(assert_hostname=False)) try: images = client.images('binstar/linux-64') except docker.errors.NotFound as err: warnings.warn("Skip Docker Tests: {}".format(err)) return False except requests.ConnectionError: warnings.warn( "Skip Docker Tests: image binstar/linux-64 is not pulled") return False return True
def docker_client(environment, version=None, tls_config=None, host=None, tls_version=None): """ Returns a docker-py client configured using environment variables according to the same logic as the official Docker client. """ try: kwargs = kwargs_from_env(environment=environment, ssl_version=tls_version) except TLSParameterError: raise UserError( "TLS configuration is invalid - make sure your DOCKER_TLS_VERIFY " "and DOCKER_CERT_PATH are set correctly.\n" "You might need to run `eval \"$(docker-machine env default)\"`") if host: kwargs['base_url'] = host if tls_config: kwargs['tls'] = tls_config if version: kwargs['version'] = version timeout = environment.get('COMPOSE_HTTP_TIMEOUT') if timeout: kwargs['timeout'] = int(timeout) else: kwargs['timeout'] = HTTP_TIMEOUT kwargs['user_agent'] = generate_user_agent() return Client(**kwargs)
def test_search(self): client = docker.APIClient(timeout=10, **kwargs_from_env()) res = client.search('busybox') self.assertTrue(len(res) >= 1) base_img = [x for x in res if x['name'] == 'busybox'] self.assertEqual(len(base_img), 1) self.assertIn('description', base_img[0])
def __init__(self): config = self._load_config() self.REDIS_HOST = config['redis_host'] self.PYWB_HOST = config['pywb_host'] self.C_EXPIRE_TIME = config['container_expire_secs'] self.Q_EXPIRE_TIME = config['queue_expire_secs'] self.REMOVE_EXP_TIME = config['remove_expired_secs'] self.VERSION = config['api_version'] self.VNC_PORT = config['vnc_port'] self.CMD_PORT = config['cmd_port'] self.image_prefix = config['image_prefix'] self.browsers = config['browsers'] self.redis = redis.StrictRedis(host=self.REDIS_HOST) self.redis.setnx('next_client', '1') if os.path.exists('/var/run/docker.sock'): self.cli = Client(base_url='unix://var/run/docker.sock', version=self.VERSION) else: kwargs = kwargs_from_env() kwargs['tls'].assert_hostname = False kwargs['version'] = self.VERSION self.cli = Client(**kwargs)
def image_by_name(img_name, images=None): """ Returns a list of image data for images which match img_name. Will optionally take a list of images from a docker.Client.images query to avoid multiple docker queries. """ i_reg, i_rep, i_tag = _decompose(img_name) # Correct for bash-style matching expressions. if not i_reg: i_reg = '*' if not i_tag: i_tag = '*' # If the images were not passed in, go get them. if images is None: c = docker.Client(**kwargs_from_env()) images = c.images(all=False) valid_images = [] for i in images: for t in i['RepoTags']: reg, rep, tag = _decompose(t) if matches(reg, i_reg) \ and matches(rep, i_rep) \ and matches(tag, i_tag): valid_images.append(i) break # Some repo after decompose end up with the img_name # at the end. i.e. rhel7/rsyslog if rep.endswith(img_name): valid_images.append(i) break return valid_images
def run(self): while (True): # Get a list with all the eventfd with self._containers_running_lock: to_select = [self._update_pipe[0]] + [d["eventlistener"].event_fd for d in self._containers_running.values() if d["eventlistener"] is not None] # Run the select() system call on all the eventfd. rlist, _, xlist = select.select(to_select, [], []) # First, handle xlist, by deleting the bad eventfds if len(xlist) != 0: if self._update_pipe[0] in xlist: raise Exception("CGroupMemoryWatcher: critical error, self._update_event_descriptor in xlist") with self._containers_running_lock: container_ids = [container_id for container_id, data in self._containers_running if (data["eventlistener"] is not None and data["eventlistener"].event_fd in rlist)] for container_id in container_ids: force_close_event_listener(self._containers_running[container_id]["eventlistener"]) self._containers_running[container_id]["eventlistener"] = None # If _update_event_descriptor is activated, just read a byte then restart select with a new # list of file descriptors if self._update_pipe[0] in rlist: os.read(self._update_pipe[0], 1) # Else, we have to kill some containers... elif len(rlist) != 0: containers_to_kill = set() with self._containers_running_lock: container_ids = [ (container_id, d["eventlistener"].event_fd, d["max_memory"]) for container_id, d in self._containers_running.iteritems() if ( d["eventlistener"] is not None and d["eventlistener"].event_fd in rlist)] for container_id, event_fd, max_memory in container_ids: # we have to read everything os.read(event_fd, 64 / 8) mem_usage = -1 try: mem_usage = self._get_max_memory_usage(container_id) except: pass if mem_usage != -1: self.logger.info("Deleting container %s as it exhausted its memory limit: %f/%f. Killing it.", container_id, mem_usage, max_memory) force_close_event_listener(self._containers_running[container_id]["eventlistener"]) self._containers_running[container_id]["eventlistener"] = None self._containers_running[container_id]["killed"] = True containers_to_kill.add(container_id) docker_connection = docker.Client(**kwargs_from_env()) for container_id in containers_to_kill: try: docker_connection.kill(container_id) except: pass
def initialize(): """ Initialize the discovery service. This is done automatically when discovery is imported, but it can be used to reinitialize the service. """ global services discovery_mode = os.getenv('DISCOVERY_MODE') docker_host = os.getenv('DOCKER_HOST') if not discovery_mode and docker_host: discovery_mode = 'docker' if not discovery_mode: discovery_mode = 'dns' if discovery_mode == 'docker': if not docker_host: raise ValueError("DOCKER_HOST not set") from docker.client import Client from docker.utils import kwargs_from_env ip = re.match(r'.*?://(.*?):\d+', docker_host).group(1) kwargs = kwargs_from_env(assert_hostname=False) client = Client(**kwargs) services = DockerRegistry(client, ip) elif discovery_mode == 'env': services = EnvironmentRegistry() elif discovery_mode == 'dns': services = DnsRegistry() else: raise ValueError("Unknown DISCOVERY_MODE: {}".format(discovery_mode))
def build_buildcontainer_image(base_path): assert_initialized(base_path) # To ensure version compatibility, we have to generate the kwargs ourselves client_kwargs = kwargs_from_env(assert_hostname=False) client = docker.AutoVersionClient(**client_kwargs) with make_temp_dir() as temp_dir: logger.info('Building Docker Engine context...') tarball_path = os.path.join(temp_dir, 'context.tar') tarball_file = open(tarball_path, 'wb') tarball = tarfile.TarFile(fileobj=tarball_file, mode='w') container_dir = os.path.normpath(os.path.join(base_path, 'ansible')) try: tarball.add(container_dir, arcname='ansible') except OSError: raise AnsibleContainerNotInitializedException() jinja_render_to_temp('ansible-dockerfile.j2', temp_dir, 'Dockerfile') tarball.add(os.path.join(temp_dir, 'Dockerfile'), arcname='Dockerfile') jinja_render_to_temp('hosts.j2', temp_dir, 'hosts', hosts=extract_hosts_from_docker_compose(base_path)) tarball.add(os.path.join(temp_dir, 'hosts'), arcname='hosts') tarball.close() tarball_file = open(tarball_path, 'rb') logger.info('Starting Docker build of Ansible Container image...') return [streamline for streamline in client.build(fileobj=tarball_file, rm=True, custom_context=True, pull=True, forcerm=True, tag='ansible-container-builder')]
def download_containers(self, to_download, current_options): """ Download the chosen containers on all the agents """ if current_options["backend"] == "local": self._display_info("Connecting to the local Docker daemon...") try: docker_connection = docker.Client(**kwargs_from_env()) except: self._display_error( "Cannot connect to local Docker daemon. Skipping download." ) return self.download_container_agent(to_download, docker_connection) elif current_options["backend"] in ["remote", "docker_machine"]: if current_options["backend"] == "remote": docker_daemons = current_options["docker_daemons"] else: docker_daemons = map( lambda x: DockerMachineJobManager.get_machine(x), current_options["machines"]) for daemon in docker_daemons: remote_host = daemon["remote_host"] remote_docker_port = daemon["remote_docker_port"] use_tls = daemon["use_tls"] if isinstance(use_tls, basestring): tls_config = docker.tls.TLSConfig( client_cert=(use_tls + '/cert.pem', use_tls + '/key.pem'), verify=use_tls + '/ca.pem') protocol = "https" elif isinstance(use_tls, dict): tls_config = docker.tls.TLSConfig( client_cert=(use_tls["cert"], use_tls["key"]), verify=use_tls["ca"]) protocol = "https" elif use_tls is True: tls_config = True protocol = "https" else: tls_config = False protocol = "http" try: docker_connection = docker.Client( base_url=protocol + "://" + remote_host + ":" + str(remote_docker_port), tls=tls_config) except: self._display_error( "Cannot connect to distant Docker daemon. Skipping download." ) continue self.download_container_agent(to_download, docker_connection) else: self._display_warning( "This installation tool does not support the backend remote_manual directly. You will have to pull the images by " "yourself. Here is the list: %s" % str(to_download))
def create_client() -> Client: """ Clients a Docker client. Will raise a `ConnectionError` if the Docker daemon is not accessible. :return: the Docker client """ global _client client = _client() if client is None: # First try looking at the environment variables for specification of the daemon's location docker_environment = kwargs_from_env(assert_hostname=False) if "base_url" in docker_environment: client = _create_client(docker_environment.get("base_url"), docker_environment.get("tls")) if client is None: raise ConnectionError( "Could not connect to the Docker daemon specified by the `DOCKER_X` environment variables: %s" % docker_environment) else: logging.info("Connected to Docker daemon specified by the environment variables") else: # Let's see if the Docker daemon is accessible via the UNIX socket client = _create_client("unix://var/run/docker.sock") if client is not None: logging.info("Connected to Docker daemon running on UNIX socket") else: raise ConnectionError( "Cannot connect to Docker - is the Docker daemon running? `$DOCKER_HOST` should be set or the " "daemon should be accessible via the standard UNIX socket.") _client = weakref.ref(client) assert isinstance(client, Client) return client
def docker_client(environment, version=None, tls_config=None, host=None): """ Returns a docker-py client configured using environment variables according to the same logic as the official Docker client. """ if 'DOCKER_CLIENT_TIMEOUT' in environment: log.warn("The DOCKER_CLIENT_TIMEOUT environment variable is deprecated. " "Please use COMPOSE_HTTP_TIMEOUT instead.") try: kwargs = kwargs_from_env(assert_hostname=False, environment=environment) except TLSParameterError: raise UserError( "TLS configuration is invalid - make sure your DOCKER_TLS_VERIFY " "and DOCKER_CERT_PATH are set correctly.\n" "You might need to run `eval \"$(docker-machine env default)\"`") if host: kwargs['base_url'] = host if tls_config: kwargs['tls'] = tls_config if version: kwargs['version'] = version timeout = environment.get('COMPOSE_HTTP_TIMEOUT') if timeout: kwargs['timeout'] = int(timeout) else: kwargs['timeout'] = HTTP_TIMEOUT return Client(**kwargs)
def client(self): if self._client is None: self._client = Client( version='auto', **kwargs_from_env(assert_hostname=self.tls_assert_hostname) ) return self._client
def _student_container_close(self, container_id, container_set): self.logger.info("Closing student container") try: docker_connection = docker.Client(**kwargs_from_env()) except: self.logger.warning("Cannot connect to Docker!") return 254 # Wait for completion return_value = 254 try: return_value = docker_connection.wait(container_id) if return_value == -1: return_value = 254 raise Exception('Container crashed!') except: pass # Verify that everything went well if self._timeout_watcher.container_had_error(container_id): return_value = 253 if self._memory_watcher.container_had_error(container_id): return_value = 252 # Remove container thread.start_new_thread(docker_connection.remove_container, (container_id, True, False, True)) container_set.remove(container_id) # Return! return return_value
def test_dump_file(tmpdir): dump_file = tmpdir.join('dump.txt') tmp = tmpdir.join('shipwright-sample') path = str(tmp) source = pkg_resources.resource_filename( __name__, 'examples/shipwright-sample', ) create_repo(path, source) client_cfg = docker_utils.kwargs_from_env() cli = docker.Client(version='1.18', **client_cfg) try: args = get_defaults() args['--dump-file'] = str(dump_file) shipw_cli.run( path=path, client_cfg=client_cfg, arguments=args, environ={}, ) assert ' : FROM busybox' in dump_file.read() finally: old_images = ( cli.images(name='shipwright/service1', quiet=True) + cli.images(name='shipwright/shared', quiet=True) + cli.images(name='shipwright/base', quiet=True) ) for image in old_images: cli.remove_image(image, force=True)
def __init__(self, name, baseimage="tryexceptpass/capsule", basetag="base"): """Make a new capsule environment""" logging.debug("Initializing") self.client = Client(**kwargs_from_env()) logging.debug("Connected client to docker socket") self.baseimage = DockerImage(baseimage, basetag, self.client) if name is None: return if self.baseimage.exists(): logging.info("Latest image is already stored locally") else: baseos = DockerImage(self.client, 'ubuntu', 'latest') if baseos.exists(): #self.baseimage = baseos.addtag('capsule', 'base') self.baseimage.build(".") else: logging.info("Latest image does not exist locally, will have to download") baseos.download() logging.debug("Image downloaded") self.baseimage.build(".") #self.baseimage = baseos.addtag('capsule', 'base') self.environment = DockerContainer(self.baseimage, name, client=self.client)
def __init__(self, docker_host='unix://var/run/docker.sock', version='auto', timeout=30, max_workers=64, assert_hostname=False, ): #kwargs = kwargs_from_env(assert_hostname=False) kwargs = kwargs_from_env(assert_hostname=assert_hostname) # environment variable DOCKER_HOST takes precedence kwargs.setdefault('base_url', docker_host) blocking_docker_client = docker.Client(version=version, timeout=timeout, **kwargs) executor = ThreadPoolExecutor(max_workers=max_workers) async_docker_client = AsyncDockerClient(blocking_docker_client, executor) self.docker_client = async_docker_client self.port = 0
def output_from_cmd(cmd, challenge, docker_version=None, docker_base_url=None, tls_settings=None): if tls_settings: tls_config = docker.tls.TLSConfig(**tls_settings) else: tls_config = None if environ.get('DOCKER_MACHINE_NAME') is None: client = docker.DockerClient(version=docker_version, base_url=docker_base_url, tls=tls_config) else: client = docker.DockerClient(**kwargs_from_env(assert_hostname=False)) b64cmd = b64encode(cmd) challenge_dir = path.join(BASE_WORKING_DIR, challenge['slug']) docker_cmd = "/ro_volume/runcmd -slug {slug} {b64cmd}".format( slug=challenge['slug'], b64cmd=b64cmd) with timeout(seconds=DOCKER_TIMEOUT): try: LOG.warn("Running `{}` in container".format(docker_cmd)) output = client.containers.run( 'registry.gitlab.com/jarv/cmdchallenge', docker_cmd, working_dir=challenge_dir, **DOCKER_OPTS) except SSLError as e: LOG.exception("SSL validation error connecting to {}".format( docker_base_url)) raise DockerValidationError("SSL Error") except ContainerError as e: LOG.exception("Container error") raise DockerValidationError( "There was a problem executing the command, return code: {}". format(e.exit_status)) except NotFound as e: LOG.exception("NotFound error") raise DockerValidationError(e.explanation) except CommandTimeoutError as e: LOG.exception("CommandTimeout error") raise DockerValidationError("Command timed out") except APIError as e: LOG.exception("Docker API error") raise DockerValidationError("Docker API error") except ConnectionError as e: LOG.exception("Docker ConnectionError") raise DockerValidationError("Docker connection error") try: output_json = json.loads(output) except ValueError as e: LOG.exception("JSON decode error") raise DockerValidationError("Command failure") if 'Error' in output_json: LOG.error("Command execution error: {}".format(output_json['Error'])) raise DockerValidationError("Command execution error") return output_json
def __init__(self): config = self._load_config() self.LOCAL_REDIS_HOST = "netcapsule_redis_1" self.REDIS_HOST = os.environ.get("REDIS_HOST", self.LOCAL_REDIS_HOST) self.PYWB_HOST = os.environ.get("PYWB_HOST", "netcapsule_pywb_1") self.C_EXPIRE_TIME = config["init_container_expire_secs"] self.Q_EXPIRE_TIME = config["queue_expire_secs"] self.REMOVE_EXP_TIME = config["remove_expired_secs"] self.VERSION = config["api_version"] self.VNC_PORT = config["vnc_port"] self.CMD_PORT = config["cmd_port"] self.MAX_CONT = config["max_containers"] self.image_prefix = config["image_prefix"] self.browser_list = config["browsers"] self.browser_paths = {} for browser in self.browser_list: path = browser["path"] if path in self.browser_paths: raise Exception("Already a browser for path {0}".format(path)) self.browser_paths[path] = browser self.default_browser = config["default_browser"] self.redirect_paths = config["redirect_paths"] self.randompages = [] try: with open(config["random_page_file"]) as fh: self.randompages = list([line.rstrip() for line in fh]) except Exception as e: print(e) self.redis = redis.StrictRedis(host=self.REDIS_HOST) self.redis.setnx("next_client", "1") self.redis.setnx("max_containers", self.MAX_CONT) throttle_samples = config["throttle_samples"] self.redis.setnx("throttle_samples", throttle_samples) throttle_max_avg = config["throttle_max_avg"] self.redis.setnx("throttle_max_avg", throttle_max_avg) self.redis.setnx("container_expire_secs", config["full_container_expire_secs"]) self.T_EXPIRE_TIME = config["throttle_expire_secs"] if os.path.exists("/var/run/docker.sock"): self.cli = Client(base_url="unix://var/run/docker.sock", version=self.VERSION) else: kwargs = kwargs_from_env(assert_hostname=False) kwargs["version"] = self.VERSION self.cli = Client(**kwargs)
def local_client(): """Returns an instance of docker.DockerClient for communicating with local docker daemon. :returns docker.DockerClient: """ kwargs = kwargs_from_env(assert_hostname=False) kwargs["timeout"] = 600 return Client(**kwargs).api
def get_docker_client(): if current_app.config.get('DOCKER_BOOT2DOCKER', False): # TODO(Dom): for some reason when using Boot2Docker we run into issues with hostname # validation. This requires more exploration, but we disable it for now return Client(**kwargs_from_env(assert_hostname=False)) else: return Client(base_url=current_app.config.get( 'DOCKER_HOST', 'unix://var/run/docker.sock'))
def _init_cli(self): if os.path.exists('/var/run/docker.sock'): self.cli = APIClient(base_url='unix://var/run/docker.sock', version=self.api_version) else: kwargs = kwargs_from_env(assert_hostname=False) kwargs['version'] = self.api_version self.cli = APIClient(**kwargs)
def test_kwargs_from_env_empty(self): os.environ.update(DOCKER_HOST='', DOCKER_CERT_PATH='', DOCKER_TLS_VERIFY='') kwargs = kwargs_from_env() self.assertEqual(None, kwargs.get('base_url')) self.assertEqual(None, kwargs.get('tls'))
def test_client_init(self): client = docker.APIClient(version='auto', **kwargs_from_env()) client_version = client._version api_version = client.version(api_version=False)['ApiVersion'] self.assertEqual(client_version, api_version) api_version_2 = client.version()['ApiVersion'] self.assertEqual(client_version, api_version_2) client.close()
def main(): arguments = old_style_arg_dict(argparser().parse_args()) return run( path=os.getcwd(), arguments=arguments, client_cfg=kwargs_from_env(), environ=os.environ, )
def client(): """ Returns a docker-py client configured using environment variables according to the same logic as the official Docker client. """ kwargs = kwargs_from_env() if 'tls' in kwargs: kwargs['tls'].assert_hostname = False return Client(version='auto', **kwargs)
def get_client(self): if not self._client: # To ensure version compatibility, we have to generate the kwargs ourselves client_kwargs = kwargs_from_env(assert_hostname=False) self._client = docker.AutoVersionClient(**client_kwargs) self.api_version = self._client.version()['ApiVersion'] # Set the version in the env so it can be used elsewhere os.environ['DOCKER_API_VERSION'] = self.api_version return self._client
def _macosx_docker_client(args): from docker.utils import kwargs_from_env kwargs = kwargs_from_env() # Read http://docker-py.readthedocs.org/en/latest/boot2docker/ kwargs['tls'].assert_hostname = False kwargs['version'] = args.api_version kwargs['timeout'] = args.http_timeout return Client(**kwargs)