def __init__(self, _exit, verbose):
# setup_client
try:
# setup client depending if running on linux or using boot2docker (osx/win)
if sys.platform == 'linux':
self.client = docker_py.Client(version='auto')
else:
# get b2d ip
self.ip = str(sh.boot2docker.ip()).strip()
try:
# try secure connection first
kw = kwargs_from_env(assert_hostname=False)
self.client = docker_py.Client(version='auto', **kw)
except docker_py.errors.DockerException as e:
# shit - some weird boot2docker, python, docker-py, requests, and ssl error
# https://github.com/docker/docker-py/issues/465
if verbose:
log.debug(e)
log.warn("Cannot connect securely to Docker, trying insecurely")
kw = kwargs_from_env(assert_hostname=False)
if 'tls' in kw:
kw['tls'].verify = False
self.client = docker_py.Client(version='auto', **kw)
except Exception as e:
if verbose:
log.error("Could not connect to Docker - try running 'docker info' first")
if sys.platform != 'linux':
log.error("Please ensure you've run 'boot2docker up' and 'boot2docker shellinit' first and have added the ENV VARs it suggests")
if _exit:
raise e
def docker_client(args):
"""
Attempts to create a docker client.
- args: The arguments parsed on the command line.
- returns: a docker-py client
"""
if _platform == 'linux' or _platform == 'linux2':
# linux
if "docker_url" in args:
return Client(base_url=args.docker_url)
else:
# TODO: test to see if this does the right thing by default.
return Client(**kwargs_from_env())
elif _platform == 'darwin':
# OS X - Assume boot2docker, and pull from that environment.
kwargs = kwargs_from_env()
if not args.strict_docker_tls:
kwargs['tls'].assert_hostname = False
return Client(**kwargs)
elif _platform == 'win32' or _platform == 'cygwin':
# Windows.
logging.fatal("Sorry, windows is not currently supported!")
sys.exit(2)
def docker_client(args):
"""
Attempts to create a docker client.
- args: The arguments parsed on the command line.
- returns: a docker-py client
"""
if _platform == 'linux' or _platform == 'linux2':
# linux
if "docker_url" in args:
return Client(
base_url=args.docker_url,
timeout=args.timeout,
version='auto')
else:
# TODO: test to see if this does the right thing by default.
return Client(
version='auto',
timeout=args.timeout,
**kwargs_from_env())
elif _platform == 'darwin':
# OS X - Assume boot2docker, and pull from that environment.
kwargs = kwargs_from_env()
if len(kwargs) == 0:
logging.error('Could not correctly pull in docker environment. '
'Try running: eval "$(docker-machine env default)"')
sys.exit(2)
if not args.strict_docker_tls:
kwargs['tls'].assert_hostname = False
return Client(version='auto', timeout=args.timeout, **kwargs)
elif _platform == 'win32' or _platform == 'cygwin':
# Windows.
logging.fatal("Sorry, windows is not currently supported!")
sys.exit(2)
def get_docker_client():
"""
Universal method to use docker.client()
"""
try:
return docker.AutoVersionClient(**kwargs_from_env())
except docker.errors.DockerException:
return docker.Client(**kwargs_from_env())
def test_auto_client(self):
client = docker.AutoVersionClient(**kwargs_from_env())
client_version = client._version
api_version = client.version(api_version=False)['ApiVersion']
self.assertEqual(client_version, api_version)
api_version_2 = client.version()['ApiVersion']
self.assertEqual(client_version, api_version_2)
client.close()
with self.assertRaises(docker.errors.DockerException):
docker.AutoVersionClient(version='1.11', **kwargs_from_env())
def setup_machine_client(self, verbose):
try:
# try secure connection first
kw = kwargs_from_env(assert_hostname=False)
return docker_py.Client(version='auto', **kw)
except docker_py.errors.DockerException as e:
# shit - some weird boot2docker, python, docker-py, requests, and ssl error
# https://github.com/docker/docker-py/issues/465
if verbose:
log.debug(e)
log.warn("Cannot connect securely to Docker, trying insecurely")
kw = kwargs_from_env(assert_hostname=False)
if 'tls' in kw:
kw['tls'].verify = False
return docker_py.Client(version='auto', **kw)
def build_buildcontainer_image(base_path):
assert_initialized(base_path)
# To ensure version compatibility, we have to generate the kwargs ourselves
client_kwargs = kwargs_from_env(assert_hostname=False)
client = docker.AutoVersionClient(**client_kwargs)
with make_temp_dir() as temp_dir:
logger.info('Building Docker Engine context...')
tarball_path = os.path.join(temp_dir, 'context.tar')
tarball_file = open(tarball_path, 'wb')
tarball = tarfile.TarFile(fileobj=tarball_file, mode='w')
container_dir = os.path.normpath(os.path.join(base_path, 'ansible'))
try:
tarball.add(container_dir, arcname='ansible')
except OSError:
raise AnsibleContainerNotInitializedException()
jinja_render_to_temp('ansible-dockerfile.j2', temp_dir, 'Dockerfile')
tarball.add(os.path.join(temp_dir, 'Dockerfile'), arcname='Dockerfile')
jinja_render_to_temp(
'hosts.j2',
temp_dir,
'hosts',
hosts=extract_hosts_from_docker_compose(base_path))
tarball.add(os.path.join(temp_dir, 'hosts'), arcname='hosts')
tarball.close()
tarball_file = open(tarball_path, 'rb')
logger.info('Starting Docker build of Ansible Container image...')
return [
streamline
for streamline in client.build(fileobj=tarball_file,
rm=True,
custom_context=True,
pull=True,
forcerm=True,
tag='ansible-container-builder')
]
def docker_client(environment,
version=None,
tls_config=None,
host=None,
tls_version=None):
"""
Returns a docker-py client configured using environment variables
according to the same logic as the official Docker client.
"""
try:
kwargs = kwargs_from_env(environment=environment,
ssl_version=tls_version)
except TLSParameterError:
raise UserError(
"TLS configuration is invalid - make sure your DOCKER_TLS_VERIFY "
"and DOCKER_CERT_PATH are set correctly.\n"
"You might need to run `eval \"$(docker-machine env default)\"`")
if host:
kwargs['base_url'] = host
if tls_config:
kwargs['tls'] = tls_config
if version:
kwargs['version'] = version
timeout = environment.get('COMPOSE_HTTP_TIMEOUT')
if timeout:
kwargs['timeout'] = int(timeout)
else:
kwargs['timeout'] = HTTP_TIMEOUT
kwargs['user_agent'] = generate_user_agent()
return APIClient(**kwargs)
def run(self):
while (True):
ltime, container_id, max_time = self._input_queue.get()
time_diff = ltime - time.time()
if time_diff > 0: # we still have to wait
self._input_queue.put((ltime, container_id, max_time))
time.sleep(min(time_diff, 5)) # wait maximum for 5 seconds
else:
try:
usage = float(get_container_cgroup("cpuacct", container_id).get_stats()['usage'])
except:
continue # container has closed
self.logger.debug("Current time usage: %f. Max: %f.", (usage / (10 ** 9)), max_time)
if (usage / (10 ** 9)) < max_time: # retry
minimum_remaining_time = math.ceil((max_time - (usage / (10 ** 9))) / self._cpu_count)
self.logger.debug("Minimum wait: %f.", minimum_remaining_time)
self._input_queue.put((time.time() + minimum_remaining_time, container_id, max_time))
else: # kill it (with fire!)
self.logger.info("Killing container %s due to timeout", container_id)
self._container_errors.add(container_id)
try:
docker_connection = docker.Client(**kwargs_from_env())
docker_connection.kill(container_id)
except:
pass
def main():
logging.basicConfig(level=logging.INFO,
format="%(message)s",
stream=sys.stdout)
args = get_args()
client = docker.APIClient(version='auto',
timeout=args.timeout,
**kwargs_from_env())
exclude_container_labels = format_exclude_labels(
args.exclude_container_label)
if args.max_container_age:
cleanup_containers(
client,
args.max_container_age,
args.dry_run,
exclude_container_labels,
)
if args.max_image_age:
exclude_set = build_exclude_set(args.exclude_image,
args.exclude_image_file)
cleanup_images(client, args.max_image_age, args.dry_run, exclude_set)
if args.dangling_volumes:
cleanup_volumes(client, args.dry_run)
def world_of_docker():
"""Pulls and starts a container from a random Docker image in the top 100."""
with open('repos.json') as data_file:
repos = json.load(data_file)
client = Client(**kwargs_from_env())
random_repo = random.choice(repos)['name']
click.echo('Hmmmmmm.... how about %s? Everybody likes %s!' %
(random_repo, random_repo))
for line in client.pull(random_repo, stream=True):
click.echo(json.loads(line)['status'])
click.echo('Now let\'s just start up a container here...')
container = client.create_container(image=random_repo)
client.start(container=container.get('Id'))
container_name = client.inspect_container(
container['Id'])['Name'].strip('/')
click.echo('Up and running! Enjoy your new %s container, %s' %
(random_repo, container_name))
def test_dump_file(tmpdir, docker_client):
dump_file = tmpdir.join('dump.txt')
tmp = tmpdir.join('shipwright-sample')
path = str(tmp)
source = pkg_resources.resource_filename(
__name__,
'examples/shipwright-sample',
)
create_repo(path, source)
client_cfg = docker_utils.kwargs_from_env()
cli = docker_client
try:
args = get_defaults()
args['--dump-file'] = str(dump_file)
shipw_cli.run(
path=path,
client_cfg=client_cfg,
arguments=args,
environ={},
)
assert ' : FROM busybox' in dump_file.read()
finally:
old_images = (cli.images(name='shipwright/service1', quiet=True) +
cli.images(name='shipwright/shared', quiet=True) +
cli.images(name='shipwright/base', quiet=True))
for image in old_images:
cli.remove_image(image, force=True)
def push_image(self):
"""Push docker image to registry"""
client = docker.APIClient(version="auto", **kwargs_from_env())
# Build a progress setup for each layer, and only emit per-layer
# info every 1.5s
layers = {}
last_emit_time = time.time()
for line in client.push(self.output_image_spec, stream=True):
progress = json.loads(line.decode("utf-8"))
if "error" in progress:
self.log.error(progress["error"], extra=dict(phase="failed"))
raise docker.errors.ImageLoadError(progress["error"])
if "id" not in progress:
continue
if "progressDetail" in progress and progress["progressDetail"]:
layers[progress["id"]] = progress["progressDetail"]
else:
layers[progress["id"]] = progress["status"]
if time.time() - last_emit_time > 1.5:
self.log.info("Pushing image\n",
extra=dict(progress=layers, phase="pushing"))
last_emit_time = time.time()
self.log.info(
"Successfully pushed {}".format(self.output_image_spec),
extra=dict(phase="pushing"),
)
def test_sample(tmpdir, capsys):
path = str(tmpdir.join('shipwright-sample'))
source = pkg_resources.resource_filename(
__name__,
'examples/shipwright-sample',
)
repo = create_repo(path, source)
tag = repo.head.ref.commit.hexsha[:12]
client_cfg = docker_utils.kwargs_from_env()
args = get_defaults()
args['images'] = True
shipw_cli.run(
path=path,
client_cfg=client_cfg,
arguments=args,
environ={},
)
out, err = capsys.readouterr()
images = {'base', 'shared', 'service1'}
tmpl = 'shipwright/{img}:{tag}'
expected = {tmpl.format(img=i, tag=tag) for i in images}
assert {l for l in out.split('\n') if l} == expected
def __init__(self, bs, worker_config, args):
Worker.__init__(self, bs, worker_config, args)
self.client = docker.Client(
version=os.environ.get('DOCKER_VERSION'),
**kwargs_from_env(assert_hostname=False)
)
log.info('Connecting to docker daemon ...')
try:
image = args.image
if ':' in image:
self.image, self.tag = image.split(':', 1)
else:
self.image, self.tag = image, None
images = self.client.images(self.image)
except ConnectionError as err:
raise errors.BinstarError(
"Docker client could not connect to daemon (is docker installed?)\n"
"You may need to set your DOCKER_HOST environment variable")
if not images:
raise errors.BinstarError(
"You do not have the docker image '{image}'\n"
"You may need to run:\n\n\tdocker pull {image}s\n".format(image=args.image))
if self.args.allow_user_images:
log.warn("Allowing users to specify docker images")
def client(self):
"""single global client instance"""
cls = self.__class__
if cls._client is None:
if self.use_docker_client_env:
kwargs = kwargs_from_env(
assert_hostname=self.tls_assert_hostname
)
client = docker.Client(version='auto', **kwargs)
else:
if self.tls:
tls_config = True
elif self.tls_verify or self.tls_ca or self.tls_client:
tls_config = docker.tls.TLSConfig(
client_cert=self.tls_client,
ca_cert=self.tls_ca,
verify=self.tls_verify,
assert_hostname = self.tls_assert_hostname)
else:
tls_config = None
docker_host = os.environ.get('DOCKER_HOST', 'unix://var/run/docker.sock')
client = docker.Client(base_url=docker_host, tls=tls_config, version='auto')
cls._client = client
return cls._client
async def postgres(loop):
tag = 'latest'
image = 'postgres'
host = '127.0.0.1'
timeout = 60
unused_tcp_port = get_free_port()
client = DockerClient(version='auto', **kwargs_from_env())
client.images.pull(image, tag=tag)
print('Stating %s:%s on %s:%s' % (image, tag, host, unused_tcp_port))
cont = client.containers.run('%s:%s' % (image, tag), detach=True,
ports={'5432/tcp': ('0.0.0.0',
unused_tcp_port)})
try:
start_time = time.time()
conn = None
while conn is None:
if start_time + timeout < time.time():
raise Exception("Initialization timeout, failed to "
"initialize postgresql container")
try:
conn = await asyncpg.connect(
'postgresql://postgres@%s:%s/postgres'
'' % (host, unused_tcp_port),
loop=loop)
except Exception as e:
time.sleep(.1)
await conn.close()
yield (host, unused_tcp_port)
finally:
cont.kill()
cont.remove()
def download_docker_image(docker_image, target_file, cache=None):
try:
from docker.client import Client
from docker.utils import kwargs_from_env
kwargs = kwargs_from_env()
kwargs['tls'].assert_hostname = False
docker_cli = Client(**kwargs)
image = docker_cli.get_image(docker_image)
image_tar = open(target_file,'w')
image_tar.write(image.data)
image_tar.close()
except Exception as e:
if cache is not None:
cached_file = os.path.join(cache, docker_image.lower().replace('/','-').replace(':','-') + '.tgz')
if os.path.isfile(cached_file):
print 'using cached version of', docker_image
urllib.urlretrieve(cached_file, target_file)
return
print >> sys.stderr, docker_image, 'not found in cache', cache
sys.exit(1)
if isinstance(e, KeyError):
print >> sys.stderr, 'docker not configured on this machine (or environment variables are not properly set)'
else:
print >> sys.stderr, docker_image, 'not found on local machine'
print >> sys.stderr, 'you must either pull the image, or download it and use the --docker-cache option'
sys.exit(1)
def test_exact(tmpdir, docker_client):
path = str(tmpdir.join('shipwright-sample'))
source = pkg_resources.resource_filename(
__name__,
'examples/shipwright-sample',
)
create_repo(path, source)
client_cfg = docker_utils.kwargs_from_env()
cli = docker_client
args = get_defaults()
args['--exact'] = [
'shipwright/base',
]
try:
shipw_cli.run(
path=path,
client_cfg=client_cfg,
arguments=args,
environ={},
)
base, = (cli.images(name='shipwright/service1') +
cli.images(name='shipwright/shared') +
cli.images(name='shipwright/base'))
assert 'shipwright/base:master' in base['RepoTags']
assert 'shipwright/base:latest' in base['RepoTags']
finally:
old_images = (cli.images(name='shipwright/base', quiet=True))
for image in old_images:
cli.remove_image(image, force=True)
def main():
args = parse_args( sys.argv[1:] )
kwargs = kwargs_from_env()
# Connect to docker
if len( kwargs.keys() ) == 0:
logger.warning( 'Unable to discover Docker settings through env' )
logger.info( 'Using {}'.format( args.docker ) )
kwargs['base_url'] = args.docker
# Connect to ECTD
etcd_client = etcd.Client( host=args.etcd_host, port=args.etcd_port )
etcd_folder = os.path.join( args.prefix, args.domain )
logger.debug( 'Announcing to {}'.format( etcd_folder ) )
# Find the matching container
docker_client = Client(**kwargs)
try:
containers = docker_client.containers()
logger.error( containers )
except Exception as e:
logger.error( e )
sys.exit( 'FAILURE - Unable to connect Docker. Is it running?' )
# Find the matching container
matching = find_matching_container( containers, args )
# Main health checking loop
while True:
announce_services( matching.items(),
etcd_folder,
etcd_client,
args.timeout,
args.ttl,
args.vulcand )
def __init__(self):
kwargs = kwargs_from_env(assert_hostname=False)
kwargs.update({'version': 'auto'})
self.client = Client(**kwargs)
try:
self.client.info()
# mount the /Volumes folder under mac, please refer to
# https://github.com/bpeng2000/SOS/wiki/SoS-Docker-guide
# for details.
self.has_volumes = False
if platform.system() == 'Darwin':
try:
# this command log in to the docker machine, check if /Volumes has been mounted,
# and try to mount it if possible. This requires users to configure
subprocess.call(
"""docker-machine ssh "{}" 'mount | grep /Volumes || {{ echo "mounting /Volumes"; sudo mount -t vboxsf Volumes /Volumes; }}' """
.format(os.environ['DOCKER_MACHINE_NAME']),
shell=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
env.logger.trace(
'Sucessfully mount /Volumes to virtual machine')
self.has_volumes = True
except Exception as e:
env.logger.trace(
'Failed to mount /Volumes to virtual machine: {}'.
format(e))
except Exception as e:
env.logger.debug('Docker client init fail: {}'.format(e))
self.client = None
def docker_client(environment, version=None, context=None, tls_version=None):
"""
Returns a docker-py client configured using environment variables
according to the same logic as the official Docker client.
"""
try:
kwargs = kwargs_from_env(environment=environment,
ssl_version=tls_version)
except TLSParameterError:
raise UserError(
"TLS configuration is invalid - make sure your DOCKER_TLS_VERIFY "
"and DOCKER_CERT_PATH are set correctly.\n"
"You might need to run `eval \"$(docker-machine env default)\"`")
if not context:
# check env for DOCKER_HOST and certs path
host = kwargs.get("base_url", None)
tls = kwargs.get("tls", None)
verify = False if not tls else tls.verify
if host:
context = Context("compose", host=host, tls=verify)
else:
context = ContextAPI.get_current_context()
if tls:
context.set_endpoint("docker",
host=host,
tls_cfg=tls,
skip_tls_verify=not verify)
if not context.is_docker_host():
raise UserError(
"The platform targeted with the current context is not supported.\n"
"Make sure the context in use targets a Docker Engine.\n")
kwargs['base_url'] = context.Host
if context.TLSConfig:
kwargs['tls'] = context.TLSConfig
if version:
kwargs['version'] = version
timeout = environment.get('COMPOSE_HTTP_TIMEOUT')
if timeout:
kwargs['timeout'] = int(timeout)
else:
kwargs['timeout'] = HTTP_TIMEOUT
kwargs['user_agent'] = generate_user_agent()
# Workaround for
# https://pyinstaller.readthedocs.io/en/v3.3.1/runtime-information.html#ld-library-path-libpath-considerations
if 'LD_LIBRARY_PATH_ORIG' in environment:
kwargs['credstore_env'] = {
'LD_LIBRARY_PATH': environment.get('LD_LIBRARY_PATH_ORIG'),
}
use_paramiko_ssh = int(environment.get('COMPOSE_PARAMIKO_SSH', 0))
client = APIClient(use_ssh_client=not use_paramiko_ssh, **kwargs)
client._original_base_url = kwargs.get('base_url')
return client
def create_local_test_data(private=False):
appyaml = {
'appname': 'blueberry',
'entrypoints': {
'web': {
'cmd': 'python app.py',
'ports': ['5000/tcp'],
},
'daemon': {
'cmd': 'python daemon.py',
},
'service': {
'cmd': 'python service.py'
},
},
'build': 'pip install -r ./requirements.txt',
}
app = App.get_or_create('blueberry', 'http://git.hunantv.com/tonic/blueberry.git', 'token')
version = app.add_version('abe23812aeb50a17a2509c02a28423462161d306')
appconfig = version.appconfig
appconfig.update(**appyaml)
appconfig.save()
group = Group.create('group', 'group')
pod = Pod.create('pod', 'pod')
pod.assigned_to_group(group)
c = docker.Client(**kwargs_from_env(assert_hostname=False))
r = c.info()
host = Host.create(pod, '192.168.59.103:2376', r['Name'], r['ID'], r['NCPU'], r['MemTotal'])
if private:
host.assigned_to_group(group)
return app, version, group, pod, host
def test_kwargs_from_env_empty(self):
os.environ.update(DOCKER_HOST='', DOCKER_CERT_PATH='')
os.environ.pop('DOCKER_TLS_VERIFY', None)
kwargs = kwargs_from_env()
self.assertEqual(None, kwargs.get('base_url'))
self.assertEqual(None, kwargs.get('tls'))
def have_docker():
if os.environ.get('NO_DOCKER_TESTS'):
return False
try:
import docker
from docker.utils import kwargs_from_env
except:
warnings.warn("Skip Docker Tests: dockerpy is not installed")
return False
client = docker.Client(
version=os.environ.get('DOCKER_VERSION'),
**kwargs_from_env(assert_hostname=False)
)
try:
images = client.images('binstar/linux-64')
except docker.errors.NotFound as err:
warnings.warn("Skip Docker Tests: {}".format(err))
return False
except requests.ConnectionError:
warnings.warn("Skip Docker Tests: image binstar/linux-64 is not pulled")
return False
return True
def __init__(
self,
docker_host='unix://var/run/docker.sock',
version='auto',
timeout=30,
max_workers=64,
assert_hostname=False,
):
#kwargs = kwargs_from_env(assert_hostname=False)
kwargs = kwargs_from_env(assert_hostname=assert_hostname)
# environment variable DOCKER_HOST takes precedence
kwargs.setdefault('base_url', docker_host)
blocking_docker_client = docker.Client(version=version,
timeout=timeout,
**kwargs)
executor = ThreadPoolExecutor(max_workers=max_workers)
async_docker_client = AsyncDockerClient(blocking_docker_client,
executor)
self.docker_client = async_docker_client
self.port = 0
def __init__(self, bs, worker_config, args):
Worker.__init__(self, bs, worker_config, args)
self.client = docker.Client(version=os.environ.get('DOCKER_VERSION'),
**kwargs_from_env(assert_hostname=False))
log.info('Connecting to docker daemon ...')
try:
image = args.image
if ':' in image:
self.image, self.tag = image.split(':', 1)
else:
self.image, self.tag = image, None
images = self.client.images(self.image)
except ConnectionError as err:
raise errors.BinstarError(
"Docker client could not connect to daemon (is docker installed?)\n"
"You may need to set your DOCKER_HOST environment variable")
if not images:
raise errors.BinstarError(
"You do not have the docker image '{image}'\n"
"You may need to run:\n\n\tdocker pull {image}s\n".format(
image=args.image))
if self.args.allow_user_images:
log.warn("Allowing users to specify docker images")
def get_client(self):
if not self._client:
# To ensure version compatibility, we have to generate the kwargs ourselves
client_kwargs = kwargs_from_env(assert_hostname=False)
self._client = docker.AutoVersionClient(**client_kwargs)
self.api_version = self._client.version()['ApiVersion']
return self._client
def have_docker():
if os.environ.get('NO_DOCKER_TESTS'):
return False
try:
import docker
from docker.utils import kwargs_from_env
except:
warnings.warn("Skip Docker Tests: dockerpy is not installed")
return False
client = docker.Client(version=os.environ.get('DOCKER_VERSION'),
**kwargs_from_env(assert_hostname=False))
try:
images = client.images('binstar/linux-64')
except docker.errors.NotFound as err:
warnings.warn("Skip Docker Tests: {}".format(err))
return False
except requests.ConnectionError:
warnings.warn(
"Skip Docker Tests: image binstar/linux-64 is not pulled")
return False
return True
def docker_client(environment, version=None, tls_config=None, host=None,
tls_version=None):
"""
Returns a docker-py client configured using environment variables
according to the same logic as the official Docker client.
"""
try:
kwargs = kwargs_from_env(environment=environment, ssl_version=tls_version)
except TLSParameterError:
raise UserError(
"TLS configuration is invalid - make sure your DOCKER_TLS_VERIFY "
"and DOCKER_CERT_PATH are set correctly.\n"
"You might need to run `eval \"$(docker-machine env default)\"`")
if host:
kwargs['base_url'] = host
if tls_config:
kwargs['tls'] = tls_config
if version:
kwargs['version'] = version
timeout = environment.get('COMPOSE_HTTP_TIMEOUT')
if timeout:
kwargs['timeout'] = int(timeout)
else:
kwargs['timeout'] = HTTP_TIMEOUT
kwargs['user_agent'] = generate_user_agent()
return Client(**kwargs)
def test_search(self):
client = docker.APIClient(timeout=10, **kwargs_from_env())
res = client.search('busybox')
self.assertTrue(len(res) >= 1)
base_img = [x for x in res if x['name'] == 'busybox']
self.assertEqual(len(base_img), 1)
self.assertIn('description', base_img[0])
def __init__(self):
config = self._load_config()
self.REDIS_HOST = config['redis_host']
self.PYWB_HOST = config['pywb_host']
self.C_EXPIRE_TIME = config['container_expire_secs']
self.Q_EXPIRE_TIME = config['queue_expire_secs']
self.REMOVE_EXP_TIME = config['remove_expired_secs']
self.VERSION = config['api_version']
self.VNC_PORT = config['vnc_port']
self.CMD_PORT = config['cmd_port']
self.image_prefix = config['image_prefix']
self.browsers = config['browsers']
self.redis = redis.StrictRedis(host=self.REDIS_HOST)
self.redis.setnx('next_client', '1')
if os.path.exists('/var/run/docker.sock'):
self.cli = Client(base_url='unix://var/run/docker.sock',
version=self.VERSION)
else:
kwargs = kwargs_from_env()
kwargs['tls'].assert_hostname = False
kwargs['version'] = self.VERSION
self.cli = Client(**kwargs)
def image_by_name(img_name, images=None):
"""
Returns a list of image data for images which match img_name. Will
optionally take a list of images from a docker.Client.images
query to avoid multiple docker queries.
"""
i_reg, i_rep, i_tag = _decompose(img_name)
# Correct for bash-style matching expressions.
if not i_reg:
i_reg = '*'
if not i_tag:
i_tag = '*'
# If the images were not passed in, go get them.
if images is None:
c = docker.Client(**kwargs_from_env())
images = c.images(all=False)
valid_images = []
for i in images:
for t in i['RepoTags']:
reg, rep, tag = _decompose(t)
if matches(reg, i_reg) \
and matches(rep, i_rep) \
and matches(tag, i_tag):
valid_images.append(i)
break
# Some repo after decompose end up with the img_name
# at the end. i.e. rhel7/rsyslog
if rep.endswith(img_name):
valid_images.append(i)
break
return valid_images
def run(self):
while (True):
# Get a list with all the eventfd
with self._containers_running_lock:
to_select = [self._update_pipe[0]] + [d["eventlistener"].event_fd for d in self._containers_running.values() if
d["eventlistener"] is not None]
# Run the select() system call on all the eventfd.
rlist, _, xlist = select.select(to_select, [], [])
# First, handle xlist, by deleting the bad eventfds
if len(xlist) != 0:
if self._update_pipe[0] in xlist:
raise Exception("CGroupMemoryWatcher: critical error, self._update_event_descriptor in xlist")
with self._containers_running_lock:
container_ids = [container_id for container_id, data in self._containers_running
if (data["eventlistener"] is not None and data["eventlistener"].event_fd in rlist)]
for container_id in container_ids:
force_close_event_listener(self._containers_running[container_id]["eventlistener"])
self._containers_running[container_id]["eventlistener"] = None
# If _update_event_descriptor is activated, just read a byte then restart select with a new
# list of file descriptors
if self._update_pipe[0] in rlist:
os.read(self._update_pipe[0], 1)
# Else, we have to kill some containers...
elif len(rlist) != 0:
containers_to_kill = set()
with self._containers_running_lock:
container_ids = [
(container_id,
d["eventlistener"].event_fd,
d["max_memory"]) for container_id,
d in self._containers_running.iteritems() if (
d["eventlistener"] is not None and d["eventlistener"].event_fd in rlist)]
for container_id, event_fd, max_memory in container_ids:
# we have to read everything
os.read(event_fd, 64 / 8)
mem_usage = -1
try:
mem_usage = self._get_max_memory_usage(container_id)
except:
pass
if mem_usage != -1:
self.logger.info("Deleting container %s as it exhausted its memory limit: %f/%f. Killing it.", container_id, mem_usage,
max_memory)
force_close_event_listener(self._containers_running[container_id]["eventlistener"])
self._containers_running[container_id]["eventlistener"] = None
self._containers_running[container_id]["killed"] = True
containers_to_kill.add(container_id)
docker_connection = docker.Client(**kwargs_from_env())
for container_id in containers_to_kill:
try:
docker_connection.kill(container_id)
except:
pass
def initialize():
"""
Initialize the discovery service. This is done automatically when discovery is imported, but it can be used
to reinitialize the service.
"""
global services
discovery_mode = os.getenv('DISCOVERY_MODE')
docker_host = os.getenv('DOCKER_HOST')
if not discovery_mode and docker_host:
discovery_mode = 'docker'
if not discovery_mode:
discovery_mode = 'dns'
if discovery_mode == 'docker':
if not docker_host:
raise ValueError("DOCKER_HOST not set")
from docker.client import Client
from docker.utils import kwargs_from_env
ip = re.match(r'.*?://(.*?):\d+', docker_host).group(1)
kwargs = kwargs_from_env(assert_hostname=False)
client = Client(**kwargs)
services = DockerRegistry(client, ip)
elif discovery_mode == 'env':
services = EnvironmentRegistry()
elif discovery_mode == 'dns':
services = DnsRegistry()
else:
raise ValueError("Unknown DISCOVERY_MODE: {}".format(discovery_mode))
def build_buildcontainer_image(base_path):
assert_initialized(base_path)
# To ensure version compatibility, we have to generate the kwargs ourselves
client_kwargs = kwargs_from_env(assert_hostname=False)
client = docker.AutoVersionClient(**client_kwargs)
with make_temp_dir() as temp_dir:
logger.info('Building Docker Engine context...')
tarball_path = os.path.join(temp_dir, 'context.tar')
tarball_file = open(tarball_path, 'wb')
tarball = tarfile.TarFile(fileobj=tarball_file,
mode='w')
container_dir = os.path.normpath(os.path.join(base_path,
'ansible'))
try:
tarball.add(container_dir, arcname='ansible')
except OSError:
raise AnsibleContainerNotInitializedException()
jinja_render_to_temp('ansible-dockerfile.j2', temp_dir, 'Dockerfile')
tarball.add(os.path.join(temp_dir, 'Dockerfile'),
arcname='Dockerfile')
jinja_render_to_temp('hosts.j2', temp_dir, 'hosts',
hosts=extract_hosts_from_docker_compose(base_path))
tarball.add(os.path.join(temp_dir, 'hosts'), arcname='hosts')
tarball.close()
tarball_file = open(tarball_path, 'rb')
logger.info('Starting Docker build of Ansible Container image...')
return [streamline for streamline in client.build(fileobj=tarball_file,
rm=True,
custom_context=True,
pull=True,
forcerm=True,
tag='ansible-container-builder')]
def image_by_name(img_name, images=None):
"""
Returns a list of image data for images which match img_name. Will
optionally take a list of images from a docker.Client.images
query to avoid multiple docker queries.
"""
i_reg, i_rep, i_tag = _decompose(img_name)
# Correct for bash-style matching expressions.
if not i_reg:
i_reg = '*'
if not i_tag:
i_tag = '*'
# If the images were not passed in, go get them.
if images is None:
c = docker.Client(**kwargs_from_env())
images = c.images(all=False)
valid_images = []
for i in images:
for t in i['RepoTags']:
reg, rep, tag = _decompose(t)
if matches(reg, i_reg) \
and matches(rep, i_rep) \
and matches(tag, i_tag):
valid_images.append(i)
break
# Some repo after decompose end up with the img_name
# at the end. i.e. rhel7/rsyslog
if rep.endswith(img_name):
valid_images.append(i)
break
return valid_images
def download_containers(self, to_download, current_options):
""" Download the chosen containers on all the agents """
if current_options["backend"] == "local":
self._display_info("Connecting to the local Docker daemon...")
try:
docker_connection = docker.Client(**kwargs_from_env())
except:
self._display_error(
"Cannot connect to local Docker daemon. Skipping download."
)
return
self.download_container_agent(to_download, docker_connection)
elif current_options["backend"] in ["remote", "docker_machine"]:
if current_options["backend"] == "remote":
docker_daemons = current_options["docker_daemons"]
else:
docker_daemons = map(
lambda x: DockerMachineJobManager.get_machine(x),
current_options["machines"])
for daemon in docker_daemons:
remote_host = daemon["remote_host"]
remote_docker_port = daemon["remote_docker_port"]
use_tls = daemon["use_tls"]
if isinstance(use_tls, basestring):
tls_config = docker.tls.TLSConfig(
client_cert=(use_tls + '/cert.pem',
use_tls + '/key.pem'),
verify=use_tls + '/ca.pem')
protocol = "https"
elif isinstance(use_tls, dict):
tls_config = docker.tls.TLSConfig(
client_cert=(use_tls["cert"], use_tls["key"]),
verify=use_tls["ca"])
protocol = "https"
elif use_tls is True:
tls_config = True
protocol = "https"
else:
tls_config = False
protocol = "http"
try:
docker_connection = docker.Client(
base_url=protocol + "://" + remote_host + ":" +
str(remote_docker_port),
tls=tls_config)
except:
self._display_error(
"Cannot connect to distant Docker daemon. Skipping download."
)
continue
self.download_container_agent(to_download, docker_connection)
else:
self._display_warning(
"This installation tool does not support the backend remote_manual directly. You will have to pull the images by "
"yourself. Here is the list: %s" % str(to_download))
def create_client() -> Client:
"""
Clients a Docker client.
Will raise a `ConnectionError` if the Docker daemon is not accessible.
:return: the Docker client
"""
global _client
client = _client()
if client is None:
# First try looking at the environment variables for specification of the daemon's location
docker_environment = kwargs_from_env(assert_hostname=False)
if "base_url" in docker_environment:
client = _create_client(docker_environment.get("base_url"), docker_environment.get("tls"))
if client is None:
raise ConnectionError(
"Could not connect to the Docker daemon specified by the `DOCKER_X` environment variables: %s"
% docker_environment)
else:
logging.info("Connected to Docker daemon specified by the environment variables")
else:
# Let's see if the Docker daemon is accessible via the UNIX socket
client = _create_client("unix://var/run/docker.sock")
if client is not None:
logging.info("Connected to Docker daemon running on UNIX socket")
else:
raise ConnectionError(
"Cannot connect to Docker - is the Docker daemon running? `$DOCKER_HOST` should be set or the "
"daemon should be accessible via the standard UNIX socket.")
_client = weakref.ref(client)
assert isinstance(client, Client)
return client
def docker_client(environment, version=None, tls_config=None, host=None):
"""
Returns a docker-py client configured using environment variables
according to the same logic as the official Docker client.
"""
if 'DOCKER_CLIENT_TIMEOUT' in environment:
log.warn("The DOCKER_CLIENT_TIMEOUT environment variable is deprecated. "
"Please use COMPOSE_HTTP_TIMEOUT instead.")
try:
kwargs = kwargs_from_env(assert_hostname=False, environment=environment)
except TLSParameterError:
raise UserError(
"TLS configuration is invalid - make sure your DOCKER_TLS_VERIFY "
"and DOCKER_CERT_PATH are set correctly.\n"
"You might need to run `eval \"$(docker-machine env default)\"`")
if host:
kwargs['base_url'] = host
if tls_config:
kwargs['tls'] = tls_config
if version:
kwargs['version'] = version
timeout = environment.get('COMPOSE_HTTP_TIMEOUT')
if timeout:
kwargs['timeout'] = int(timeout)
else:
kwargs['timeout'] = HTTP_TIMEOUT
return Client(**kwargs)
def client(self):
"""single global client instance"""
cls = self.__class__
if cls._client is None:
if self.use_docker_client_env:
kwargs = kwargs_from_env(
assert_hostname=self.tls_assert_hostname
)
client = docker.Client(version='auto', **kwargs)
else:
if self.tls:
tls_config = True
elif self.tls_verify or self.tls_ca or self.tls_client:
tls_config = docker.tls.TLSConfig(
client_cert=self.tls_client,
ca_cert=self.tls_ca,
verify=self.tls_verify,
assert_hostname = self.tls_assert_hostname)
else:
tls_config = None
docker_host = os.environ.get('DOCKER_HOST', 'unix://var/run/docker.sock')
client = docker.Client(base_url=docker_host, tls=tls_config, version='auto')
cls._client = client
return cls._client
def client(self):
if self._client is None:
self._client = Client(
version='auto',
**kwargs_from_env(assert_hostname=self.tls_assert_hostname)
)
return self._client
def get_client(self):
if not self._client:
# To ensure version compatibility, we have to generate the kwargs ourselves
client_kwargs = kwargs_from_env(assert_hostname=False)
self._client = docker.AutoVersionClient(**client_kwargs)
self.api_version = self._client.version()['ApiVersion']
return self._client
def _student_container_close(self, container_id, container_set):
self.logger.info("Closing student container")
try:
docker_connection = docker.Client(**kwargs_from_env())
except:
self.logger.warning("Cannot connect to Docker!")
return 254
# Wait for completion
return_value = 254
try:
return_value = docker_connection.wait(container_id)
if return_value == -1:
return_value = 254
raise Exception('Container crashed!')
except:
pass
# Verify that everything went well
if self._timeout_watcher.container_had_error(container_id):
return_value = 253
if self._memory_watcher.container_had_error(container_id):
return_value = 252
# Remove container
thread.start_new_thread(docker_connection.remove_container, (container_id, True, False, True))
container_set.remove(container_id)
# Return!
return return_value
def test_dump_file(tmpdir):
dump_file = tmpdir.join('dump.txt')
tmp = tmpdir.join('shipwright-sample')
path = str(tmp)
source = pkg_resources.resource_filename(
__name__,
'examples/shipwright-sample',
)
create_repo(path, source)
client_cfg = docker_utils.kwargs_from_env()
cli = docker.Client(version='1.18', **client_cfg)
try:
args = get_defaults()
args['--dump-file'] = str(dump_file)
shipw_cli.run(
path=path,
client_cfg=client_cfg,
arguments=args,
environ={},
)
assert ' : FROM busybox' in dump_file.read()
finally:
old_images = (
cli.images(name='shipwright/service1', quiet=True) +
cli.images(name='shipwright/shared', quiet=True) +
cli.images(name='shipwright/base', quiet=True)
)
for image in old_images:
cli.remove_image(image, force=True)
def download_docker_image(docker_image, target_file, cache=None):
try:
from docker.client import Client
from docker.utils import kwargs_from_env
kwargs = kwargs_from_env()
kwargs['tls'].assert_hostname = False
docker_cli = Client(**kwargs)
image = docker_cli.get_image(docker_image)
image_tar = open(target_file,'w')
image_tar.write(image.data)
image_tar.close()
except Exception as e:
if cache is not None:
cached_file = os.path.join(cache, docker_image.lower().replace('/','-').replace(':','-') + '.tgz')
if os.path.isfile(cached_file):
print 'using cached version of', docker_image
urllib.urlretrieve(cached_file, target_file)
return
print >> sys.stderr, docker_image, 'not found in cache', cache
sys.exit(1)
if isinstance(e, KeyError):
print >> sys.stderr, 'docker not configured on this machine (or environment variables are not properly set)'
else:
print >> sys.stderr, docker_image, 'not found on local machine'
print >> sys.stderr, 'you must either pull the image, or download it and use the --docker-cache option'
sys.exit(1)
def __init__(self, name, baseimage="tryexceptpass/capsule", basetag="base"):
"""Make a new capsule environment"""
logging.debug("Initializing")
self.client = Client(**kwargs_from_env())
logging.debug("Connected client to docker socket")
self.baseimage = DockerImage(baseimage, basetag, self.client)
if name is None:
return
if self.baseimage.exists():
logging.info("Latest image is already stored locally")
else:
baseos = DockerImage(self.client, 'ubuntu', 'latest')
if baseos.exists():
#self.baseimage = baseos.addtag('capsule', 'base')
self.baseimage.build(".")
else:
logging.info("Latest image does not exist locally, will have to download")
baseos.download()
logging.debug("Image downloaded")
self.baseimage.build(".")
#self.baseimage = baseos.addtag('capsule', 'base')
self.environment = DockerContainer(self.baseimage, name, client=self.client)
def __init__(self,
docker_host='unix://var/run/docker.sock',
version='auto',
timeout=30,
max_workers=64,
assert_hostname=False,
):
#kwargs = kwargs_from_env(assert_hostname=False)
kwargs = kwargs_from_env(assert_hostname=assert_hostname)
# environment variable DOCKER_HOST takes precedence
kwargs.setdefault('base_url', docker_host)
blocking_docker_client = docker.Client(version=version,
timeout=timeout,
**kwargs)
executor = ThreadPoolExecutor(max_workers=max_workers)
async_docker_client = AsyncDockerClient(blocking_docker_client,
executor)
self.docker_client = async_docker_client
self.port = 0
def output_from_cmd(cmd,
challenge,
docker_version=None,
docker_base_url=None,
tls_settings=None):
if tls_settings:
tls_config = docker.tls.TLSConfig(**tls_settings)
else:
tls_config = None
if environ.get('DOCKER_MACHINE_NAME') is None:
client = docker.DockerClient(version=docker_version,
base_url=docker_base_url,
tls=tls_config)
else:
client = docker.DockerClient(**kwargs_from_env(assert_hostname=False))
b64cmd = b64encode(cmd)
challenge_dir = path.join(BASE_WORKING_DIR, challenge['slug'])
docker_cmd = "/ro_volume/runcmd -slug {slug} {b64cmd}".format(
slug=challenge['slug'], b64cmd=b64cmd)
with timeout(seconds=DOCKER_TIMEOUT):
try:
LOG.warn("Running `{}` in container".format(docker_cmd))
output = client.containers.run(
'registry.gitlab.com/jarv/cmdchallenge',
docker_cmd,
working_dir=challenge_dir,
**DOCKER_OPTS)
except SSLError as e:
LOG.exception("SSL validation error connecting to {}".format(
docker_base_url))
raise DockerValidationError("SSL Error")
except ContainerError as e:
LOG.exception("Container error")
raise DockerValidationError(
"There was a problem executing the command, return code: {}".
format(e.exit_status))
except NotFound as e:
LOG.exception("NotFound error")
raise DockerValidationError(e.explanation)
except CommandTimeoutError as e:
LOG.exception("CommandTimeout error")
raise DockerValidationError("Command timed out")
except APIError as e:
LOG.exception("Docker API error")
raise DockerValidationError("Docker API error")
except ConnectionError as e:
LOG.exception("Docker ConnectionError")
raise DockerValidationError("Docker connection error")
try:
output_json = json.loads(output)
except ValueError as e:
LOG.exception("JSON decode error")
raise DockerValidationError("Command failure")
if 'Error' in output_json:
LOG.error("Command execution error: {}".format(output_json['Error']))
raise DockerValidationError("Command execution error")
return output_json
def __init__(self):
config = self._load_config()
self.LOCAL_REDIS_HOST = "netcapsule_redis_1"
self.REDIS_HOST = os.environ.get("REDIS_HOST", self.LOCAL_REDIS_HOST)
self.PYWB_HOST = os.environ.get("PYWB_HOST", "netcapsule_pywb_1")
self.C_EXPIRE_TIME = config["init_container_expire_secs"]
self.Q_EXPIRE_TIME = config["queue_expire_secs"]
self.REMOVE_EXP_TIME = config["remove_expired_secs"]
self.VERSION = config["api_version"]
self.VNC_PORT = config["vnc_port"]
self.CMD_PORT = config["cmd_port"]
self.MAX_CONT = config["max_containers"]
self.image_prefix = config["image_prefix"]
self.browser_list = config["browsers"]
self.browser_paths = {}
for browser in self.browser_list:
path = browser["path"]
if path in self.browser_paths:
raise Exception("Already a browser for path {0}".format(path))
self.browser_paths[path] = browser
self.default_browser = config["default_browser"]
self.redirect_paths = config["redirect_paths"]
self.randompages = []
try:
with open(config["random_page_file"]) as fh:
self.randompages = list([line.rstrip() for line in fh])
except Exception as e:
print(e)
self.redis = redis.StrictRedis(host=self.REDIS_HOST)
self.redis.setnx("next_client", "1")
self.redis.setnx("max_containers", self.MAX_CONT)
throttle_samples = config["throttle_samples"]
self.redis.setnx("throttle_samples", throttle_samples)
throttle_max_avg = config["throttle_max_avg"]
self.redis.setnx("throttle_max_avg", throttle_max_avg)
self.redis.setnx("container_expire_secs", config["full_container_expire_secs"])
self.T_EXPIRE_TIME = config["throttle_expire_secs"]
if os.path.exists("/var/run/docker.sock"):
self.cli = Client(base_url="unix://var/run/docker.sock", version=self.VERSION)
else:
kwargs = kwargs_from_env(assert_hostname=False)
kwargs["version"] = self.VERSION
self.cli = Client(**kwargs)
def local_client():
"""Returns an instance of docker.DockerClient for communicating with
local docker daemon.
:returns docker.DockerClient:
"""
kwargs = kwargs_from_env(assert_hostname=False)
kwargs["timeout"] = 600
return Client(**kwargs).api
def get_docker_client():
if current_app.config.get('DOCKER_BOOT2DOCKER', False):
# TODO(Dom): for some reason when using Boot2Docker we run into issues with hostname
# validation. This requires more exploration, but we disable it for now
return Client(**kwargs_from_env(assert_hostname=False))
else:
return Client(base_url=current_app.config.get(
'DOCKER_HOST', 'unix://var/run/docker.sock'))
def _init_cli(self):
if os.path.exists('/var/run/docker.sock'):
self.cli = APIClient(base_url='unix://var/run/docker.sock',
version=self.api_version)
else:
kwargs = kwargs_from_env(assert_hostname=False)
kwargs['version'] = self.api_version
self.cli = APIClient(**kwargs)
def test_kwargs_from_env_empty(self):
os.environ.update(DOCKER_HOST='',
DOCKER_CERT_PATH='',
DOCKER_TLS_VERIFY='')
kwargs = kwargs_from_env()
self.assertEqual(None, kwargs.get('base_url'))
self.assertEqual(None, kwargs.get('tls'))
def test_client_init(self):
client = docker.APIClient(version='auto', **kwargs_from_env())
client_version = client._version
api_version = client.version(api_version=False)['ApiVersion']
self.assertEqual(client_version, api_version)
api_version_2 = client.version()['ApiVersion']
self.assertEqual(client_version, api_version_2)
client.close()
def main():
arguments = old_style_arg_dict(argparser().parse_args())
return run(
path=os.getcwd(),
arguments=arguments,
client_cfg=kwargs_from_env(),
environ=os.environ,
)
def client():
"""
Returns a docker-py client configured using environment variables
according to the same logic as the official Docker client.
"""
kwargs = kwargs_from_env()
if 'tls' in kwargs:
kwargs['tls'].assert_hostname = False
return Client(version='auto', **kwargs)
def get_client(self):
if not self._client:
# To ensure version compatibility, we have to generate the kwargs ourselves
client_kwargs = kwargs_from_env(assert_hostname=False)
self._client = docker.AutoVersionClient(**client_kwargs)
self.api_version = self._client.version()['ApiVersion']
# Set the version in the env so it can be used elsewhere
os.environ['DOCKER_API_VERSION'] = self.api_version
return self._client
def _macosx_docker_client(args):
from docker.utils import kwargs_from_env
kwargs = kwargs_from_env()
# Read http://docker-py.readthedocs.org/en/latest/boot2docker/
kwargs['tls'].assert_hostname = False
kwargs['version'] = args.api_version
kwargs['timeout'] = args.http_timeout
return Client(**kwargs)