def docker_client(args):
"""
Attempts to create a docker client.
- args: The arguments parsed on the command line.
- returns: a docker-py client
"""
if _platform == 'linux' or _platform == 'linux2':
# linux
if "docker_url" in args:
return Client(
base_url=args.docker_url,
timeout=args.timeout,
version='auto')
else:
# TODO: test to see if this does the right thing by default.
return Client(
version='auto',
timeout=args.timeout,
**kwargs_from_env())
elif _platform == 'darwin':
# OS X - Assume boot2docker, and pull from that environment.
kwargs = kwargs_from_env()
if len(kwargs) == 0:
logging.error('Could not correctly pull in docker environment. '
'Try running: eval "$(docker-machine env default)"')
sys.exit(2)
if not args.strict_docker_tls:
kwargs['tls'].assert_hostname = False
return Client(version='auto', timeout=args.timeout, **kwargs)
elif _platform == 'win32' or _platform == 'cygwin':
# Windows.
logging.fatal("Sorry, windows is not currently supported!")
sys.exit(2)
def docker_client(args):
"""
Attempts to create a docker client.
- args: The arguments parsed on the command line.
- returns: a docker-py client
"""
if _platform == 'linux' or _platform == 'linux2':
# linux
if "docker_url" in args:
return Client(base_url=args.docker_url)
else:
# TODO: test to see if this does the right thing by default.
return Client(**kwargs_from_env())
elif _platform == 'darwin':
# OS X - Assume boot2docker, and pull from that environment.
kwargs = kwargs_from_env()
if not args.strict_docker_tls:
kwargs['tls'].assert_hostname = False
return Client(**kwargs)
elif _platform == 'win32' or _platform == 'cygwin':
# Windows.
logging.fatal("Sorry, windows is not currently supported!")
sys.exit(2)
def _init_cli(self):
if os.path.exists('/var/run/docker.sock'):
self.cli = Client(base_url='unix://var/run/docker.sock',
version=self.api_version)
else:
kwargs = kwargs_from_env(assert_hostname=False)
kwargs['version'] = self.api_version
self.cli = Client(**kwargs)
def download_docker_image(docker_image, target_file):
from docker.client import Client
try: # First attempt boot2docker, because it is fail-fast
from docker.utils import kwargs_from_env
kwargs = kwargs_from_env()
kwargs['tls'].assert_hostname = False
docker_cli = Client(**kwargs)
except KeyError as e: # Assume this means we are not using boot2docker
docker_cli = Client(base_url='unix://var/run/docker.sock', tls=False)
image = docker_cli.get_image(docker_image)
image_tar = open(target_file, 'w')
image_tar.write(image.data)
image_tar.close()
def connect(self):
url = self.api_cfg.get('url', 'unix://var/run/docker.sock')
version = self.api_cfg.get('version', '1.12')
boot2docker = self.api_cfg.get('boot2docker')
tls_config = self._construct_tlsconfig()
if boot2docker is True:
kwargs = kwargs_from_env()
kwargs['tls'].assert_hostname = False
client = Client(**kwargs)
else:
client = Client(base_url=url,
version=str(version),
tls=tls_config,
timeout=50)
return client
def run(self, ssl_env_name='STUNNEL_SSL'):
"""
Run a container scan for a variable containing a certificate env dictionary
Args:
ssl_env_name(string): The string containing the certificate env
"""
cli = Client(base_url='unix://var/run/docker.sock')
for container in cli.containers():
container_details = cli.inspect_container(container.get('Id'))
container_envs = container_details.get('Config').get('Env')
env_ssl = [env for env in container_envs if ssl_env_name in env]
if len(env_ssl) > 0:
env_cert = env_ssl[0].split('=', 1)[1]
env_json = json.loads(env_cert)
raw_ssl = env_json.get('cert')
cert = c.load_certificate(c.FILETYPE_PEM, raw_ssl)
not_after = cert.get_notAfter()
not_after_date = self.get_cert_time(not_after)
has_expired = cert.has_expired()
signature_algorithm = cert.get_signature_algorithm()
self.logger.info("Found stunnel container envs",
extra={
'notAfter':
'{}'.format(not_after),
'notAfterDate':
'{}'.format(not_after_date),
'hasExpired':
'{}'.format(has_expired),
'containerId':
'{}'.format(container.get('Id')),
'signatureAlgorithm':
'{}'.format(signature_algorithm)
})
def docker_client():
# MAC OSX SUPPORT
# if platform.system() == 'Darwin':
# kwargs = kwargs_from_env(assert_hostname=False)
# return Client(**kwargs)
# else:
return Client(version='auto')
def test_update_headers(self):
sample_headers = {
'X-Docker-Locale': 'en-US',
}
def f(self, headers=None):
return headers
client = Client()
client._auth_configs = {}
g = update_headers(f)
assert g(client, headers=None) is None
assert g(client, headers={}) == {}
assert g(client, headers={'Content-type': 'application/json'}) == {
'Content-type': 'application/json',
}
client._auth_configs = {'HttpHeaders': sample_headers}
assert g(client, headers=None) == sample_headers
assert g(client, headers={}) == sample_headers
assert g(client, headers={'Content-type': 'application/json'}) == {
'Content-type': 'application/json',
'X-Docker-Locale': 'en-US',
}
def download_docker_image(docker_image, target_file, cache=None):
try:
from docker.client import Client
from docker.utils import kwargs_from_env
kwargs = kwargs_from_env()
kwargs['tls'] = False
docker_cli = Client(**kwargs)
image = docker_cli.get_image(docker_image)
image_tar = open(target_file,'w')
image_tar.write(image.data)
image_tar.close()
except Exception as e:
if cache is not None:
cached_file = os.path.join(cache, docker_image.lower().replace('/','-').replace(':','-') + '.tgz')
if os.path.isfile(cached_file):
print 'using cached version of', docker_image
urllib.urlretrieve(cached_file, target_file)
return
print >> sys.stderr, docker_image, 'not found in cache', cache
sys.exit(1)
if isinstance(e, KeyError):
print >> sys.stderr, 'docker not configured on this machine (or environment variables are not properly set)'
else:
print >> sys.stderr, docker_image, 'not found on local machine'
print >> sys.stderr, 'you must either pull the image, or download it and use the --docker-cache option'
sys.exit(1)
def initialize():
"""
Initialize the discovery service. This is done automatically when discovery is imported, but it can be used
to reinitialize the service.
"""
global services
discovery_mode = os.getenv('DISCOVERY_MODE')
docker_host = os.getenv('DOCKER_HOST')
if not discovery_mode and docker_host:
discovery_mode = 'docker'
if not discovery_mode:
discovery_mode = 'dns'
if discovery_mode == 'docker':
if not docker_host:
raise ValueError("DOCKER_HOST not set")
from docker.client import Client
from docker.utils import kwargs_from_env
ip = re.match(r'.*?://(.*?):\d+', docker_host).group(1)
kwargs = kwargs_from_env(assert_hostname=False)
client = Client(**kwargs)
services = DockerRegistry(client, ip)
elif discovery_mode == 'env':
services = EnvironmentRegistry()
elif discovery_mode == 'dns':
services = DnsRegistry()
else:
raise ValueError("Unknown DISCOVERY_MODE: {}".format(discovery_mode))
def world_of_docker():
"""Pulls and starts a container from a random Docker image in the top 100."""
with open('repos.json') as data_file:
repos = json.load(data_file)
client = Client(**kwargs_from_env())
random_repo = random.choice(repos)['name']
click.echo('Hmmmmmm.... how about %s? Everybody likes %s!' %
(random_repo, random_repo))
for line in client.pull(random_repo, stream=True):
click.echo(json.loads(line)['status'])
click.echo('Now let\'s just start up a container here...')
container = client.create_container(image=random_repo)
client.start(container=container.get('Id'))
container_name = client.inspect_container(
container['Id'])['Name'].strip('/')
click.echo('Up and running! Enjoy your new %s container, %s' %
(random_repo, container_name))
def client():
"""
Returns a docker-py client configured using environment variables
according to the same logic as the official Docker client.
"""
kwargs = kwargs_from_env()
if 'tls' in kwargs:
kwargs['tls'].assert_hostname = False
return Client(version='auto', **kwargs)
def get_cli(assert_hostname=True):
kwargs = kwargs_from_env()
if not assert_hostname:
kwargs['tls'].assert_hostname = False
cli = Client(**kwargs)
return cli
def docker_client():
"""Create a Docker client instance or return an existing one."""
global _DOCKER_CLIENT # pylint: disable = global-statement
if _DOCKER_CLIENT:
return _DOCKER_CLIENT
else:
# assert_hostname=False is required when using boot2docker, it's taken as a hint from Fig: https://github.com/docker/fig/blob/master/compose/cli/docker_client.py#L29
_DOCKER_CLIENT = Client(**kwargs_from_env(assert_hostname=False))
return _DOCKER_CLIENT
def main(self, args):
# !! TODO needs to implement login if using that
containers = []
try:
directory = args.metadata_path
directory = os.path.expanduser(directory)
with open(os.path.join(directory, "containers"), 'r') as f:
for line in f:
container = ast.literal_eval(line.rstrip("\n"))
containers.append(container['container_id'] + "," +
container['host'])
except:
pass
host_args = Object()
host_args.metadata_path = args.metadata_path
host_args.z = True
host_a = hosts.hosts.main(host_args)
host_c = []
for host in host_a:
# !! TODO is using TLS, put in env for each host
c = Client(**kwargs_from_env())
#tls_config = docker.tls.TLSConfig(verify=False)
#c = docker.Client(base_url='tcp://'+host, version='1.12',
# #tls=tls_config,
# timeout=2)
host_c.append(c.containers())
compare_containers = []
try:
for container in host_c[0]:
compare_containers.append(container['Id'])
except:
if not args.z:
print "no hosts found"
return ""
running_containers = []
for item in containers:
container_id = item.split(',')[0]
if container_id in compare_containers:
running_containers.append(item)
if not args.z:
for container in running_containers:
print container
return running_containers
def _init_docker():
kwargs = kwargs_from_env()
if 'tls' in kwargs:
# see http://docker-py.readthedocs.org/en/latest/boot2docker/
import requests.packages.urllib3 as urllib3
urllib3.disable_warnings()
kwargs['tls'].assert_hostname = False
docker = Client(**kwargs)
try:
docker.version()
except:
raise UserMessageException("Please set up 'docker' correctly")
return docker
def get_client(daemon_client):
"""Get client.
Returns docker client using daemon_client as configuration.
:param daemon_client: optional configuration for client creation
:raises NonRecoverableError:
when docker.errors.APIError during client.
:return: docker client
"""
try:
return Client(**daemon_client)
except DockerException as e:
raise NonRecoverableError(
'Error while getting client: {0}.'.format(str(e)))
def client_from_opts(client_opts=()):
"""
Construct a docker-py Client from a string specifying options.
"""
kwargs = CLIENT_DEFAULTS.copy()
for name, value in client_opts:
if name == "timeout":
kwargs["timeout"] = int(value)
elif name == "tls":
kwargs["tls"] = strtobool(value.lower())
else:
kwargs[name] = value
return Client(**kwargs)
def test_kwargs_from_env_tls(self):
os.environ.update(DOCKER_HOST='tcp://192.168.59.103:2376',
DOCKER_CERT_PATH=TEST_CERT_DIR,
DOCKER_TLS_VERIFY='1')
kwargs = kwargs_from_env(assert_hostname=False)
self.assertEqual('https://192.168.59.103:2376', kwargs['base_url'])
self.assertTrue('ca.pem' in kwargs['tls'].verify)
self.assertTrue('cert.pem' in kwargs['tls'].cert[0])
self.assertTrue('key.pem' in kwargs['tls'].cert[1])
self.assertEqual(False, kwargs['tls'].assert_hostname)
try:
client = Client(**kwargs)
self.assertEqual(kwargs['base_url'], client.base_url)
self.assertEqual(kwargs['tls'].verify, client.verify)
self.assertEqual(kwargs['tls'].cert, client.cert)
except TypeError as e:
self.fail(e)
def cli(ctx, url, network):
# initialize Client
cl = Client(base_url=url, version='auto')
# output version to show the connected succeeded
v = cl.version()
info('Connected to Docker {v[Version]}, api version '
'{v[ApiVersion]}.'.format(v=v))
# find frontend network
nets = [n for n in cl.networks(names=[network]) if n['Name'] == network]
assert len(nets) < 2 # WTF?
if not nets:
exit_err("Could not find a network name {!r}".format(network))
ctx.obj = {'cl': cl, 'network_name': network, 'network': nets[0]['Name']}
def test_kwargs_from_env_tls_verify_false_no_cert(self):
temp_dir = tempfile.mkdtemp()
cert_dir = os.path.join(temp_dir, '.docker')
shutil.copytree(TEST_CERT_DIR, cert_dir)
os.environ.update(DOCKER_HOST='tcp://192.168.59.103:2376',
HOME=temp_dir,
DOCKER_TLS_VERIFY='')
os.environ.pop('DOCKER_CERT_PATH', None)
kwargs = kwargs_from_env(assert_hostname=True)
self.assertEqual('https://192.168.59.103:2376', kwargs['base_url'])
self.assertTrue('ca.pem' in kwargs['tls'].ca_cert)
self.assertTrue('cert.pem' in kwargs['tls'].cert[0])
self.assertTrue('key.pem' in kwargs['tls'].cert[1])
self.assertEqual(True, kwargs['tls'].assert_hostname)
self.assertEqual(False, kwargs['tls'].verify)
try:
client = Client(**kwargs)
self.assertEqual(kwargs['base_url'], client.base_url)
self.assertEqual(kwargs['tls'].cert, client.cert)
self.assertFalse(kwargs['tls'].verify)
except TypeError as e:
self.fail(e)
def __init__(self, project):
self.project = project
self.checkout_directory = project.get_checkout_directory()
self.artifact_directory = project.get_artifact_directory()
self.serve_directory = project.get_serve_directory()
self.image_name = self.get_image_name()
self.container_name = self.get_container_name()
kwargs = kwargs_from_env()
if settings.DEBUG and osx:
# development helper for boot2docker users
kwargs['tls'].assert_hostname = False
self.docker = Client(**kwargs)
if not os.path.exists(self.checkout_directory):
raise NonexistantCheckout(
'No such checkout: %s' % self.checkout_directory
)
if not os.path.exists(self.artifact_directory):
os.makedirs(self.artifact_directory)
if not os.path.exists(self.serve_directory):
os.makedirs(self.serve_directory)
def getImages(self, daemon_address=None):
c = Client(daemon_address)
result = Bag()
for i, image in enumerate(c.images()):
result['r_%i' % i] = Bag(image)
return result
def __init__(self):
config = self._load_config()
self.LOCAL_REDIS_HOST = 'netcapsule_redis_1'
self.REDIS_HOST = os.environ.get('REDIS_HOST', self.LOCAL_REDIS_HOST)
self.PYWB_HOST = os.environ.get('PYWB_HOST', 'netcapsule_pywb_1')
self.C_EXPIRE_TIME = config['init_container_expire_secs']
self.Q_EXPIRE_TIME = config['queue_expire_secs']
self.REMOVE_EXP_TIME = config['remove_expired_secs']
self.VERSION = config['api_version']
self.VNC_PORT = config['vnc_port']
self.CMD_PORT = config['cmd_port']
self.MAX_CONT = config['max_containers']
self.image_prefix = config['image_prefix']
self.browser_list = config['browsers']
self.browser_paths = {}
for browser in self.browser_list:
path = browser['path']
if path in self.browser_paths:
raise Exception('Already a browser for path {0}'.format(path))
self.browser_paths[path] = browser
self.default_browser = config['default_browser']
self.redirect_paths = config['redirect_paths']
self.randompages = []
try:
with open(config['random_page_file']) as fh:
self.randompages = list([line.rstrip() for line in fh])
except Exception as e:
print(e)
self.redis = redis.StrictRedis(host=self.REDIS_HOST)
self.redis.setnx('next_client', '1')
self.redis.setnx('max_containers', self.MAX_CONT)
self.redis.setnx('num_containers', '0')
self.redis.setnx('cpu_auto_adjust', 5.5)
throttle_samples = config['throttle_samples']
self.redis.setnx('throttle_samples', throttle_samples)
throttle_max_avg = config['throttle_max_avg']
self.redis.setnx('throttle_max_avg', throttle_max_avg)
self.redis.setnx('container_expire_secs',
config['full_container_expire_secs'])
self.T_EXPIRE_TIME = config['throttle_expire_secs']
if os.path.exists('/var/run/docker.sock'):
self.cli = Client(base_url='unix://var/run/docker.sock',
version=self.VERSION)
else:
kwargs = kwargs_from_env(assert_hostname=False)
kwargs['version'] = self.VERSION
self.cli = Client(**kwargs)
def __init__(self, remote, username=None, password=None, email=None):
super(DockerPyClient,self).__init__()
self.client = Client(base_url=remote, version='1.15')
if username:
self.client.login(username=username, password=password, email=email)
def __init__(self):
base_url = self.dockerconf.get('base_url', '')
self.client = Client(base_url=base_url)
logger.debug("Client init")
self.check = None
import sys
import requests
import re
import unicodedata
os.environ['NO_PROXY'] = '127.0.0.1'
UUID_HASH = uuid.UUID('12345678123456781234567812345678')
logger = logging.getLogger(__name__)
logging.basicConfig(format="%(asctime)s %(levelname)s %(name)s - %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
stream=sys.stdout,
level=logging.WARNING)
if os.path.exists('/rootfs/var/run/docker.sock'):
docker_cli = Client(base_url='unix://rootfs/var/run/docker.sock',
version='auto')
else:
docker_cli = Client(**kwargs_from_env(assert_hostname=False))
def get_request_headers(ctx):
return {
"Content-type": "application/json",
"Authorization": "Bearer " + ctx['api_key']
}
def hash_id(id):
return str(uuid.uuid5(UUID_HASH, id))
def setUp(self):
self.client = Client()
def download(url, filename, cache=None):
if cache is not None:
basename = os.path.basename(filename)
cachename = os.path.join(cache, basename)
if os.path.isfile(cachename):
print('- using cached version of', basename)
shutil.copy(cachename, filename)
return
# Special url to find a file associated with a github release.
# github://cf-platform-eng/meta-buildpack/meta-buildpack.tgz
# will find the file named meta-buildpack-0.0.3.tgz in the latest
# release for https://github.com/cf-platform-eng/meta-buildpack
if url.startswith("github:"):
repo_name = url.lstrip("github:").lstrip("/").lstrip("/")
file_name = os.path.basename(repo_name)
repo_name = os.path.dirname(repo_name)
url = "https://api.github.com/repos/" + repo_name + "/releases/latest"
response = requests.get(url, stream=True)
response.raise_for_status()
release = response.json()
assets = release.get('assets', [])
url = None
pattern = re.compile('.*\\.'.join(file_name.rsplit('.', 1))+'\\Z')
for asset in assets:
if pattern.match(asset['name']) is not None:
url = asset['browser_download_url']
break
if url is None:
print('no matching asset found for repo', repo_name, 'file', file_name, file=sys.stderr)
sys.exit(1)
# Fallthrough intentional, we now proceed to download the URL we found
if url.startswith("http:") or url.startswith("https"):
# [mboldt:20160908] Using urllib.urlretrieve gave an "Access
# Denied" page when trying to download docker boshrelease.
# I don't know why. requests.get works. Do what works.
response = requests.get(url, stream=True)
response.raise_for_status()
with open(filename, 'wb') as file:
for chunk in response.iter_content(chunk_size=1024):
if chunk:
file.write(chunk)
elif url.startswith("docker:"):
docker_image = url.lstrip("docker:").lstrip("/").lstrip("/")
try:
from docker.client import Client
from docker.utils import kwargs_from_env
kwargs = kwargs_from_env()
kwargs['tls'] = False
docker_cli = Client(**kwargs)
image = docker_cli.get_image(docker_image)
image_tar = open(filename,'w')
image_tar.write(image.data)
image_tar.close()
except KeyError as e:
print('docker not configured on this machine (or environment variables are not properly set)', file=sys.stderr)
sys.exit(1)
except:
print(docker_image, 'not found on local machine', file=sys.stderr)
print('you must either pull the image, or download it and use the --cache option', file=sys.stderr)
sys.exit(1)
elif os.path.isdir(url):
shutil.copytree(url, filename)
else:
shutil.copy(url, filename)
#!/usr/bin/env python import dockerpty import os from docker.client import Client from docker.utils import kwargs_from_env client = Client(base_url=os.environ['DOCKER_HOST'], version=os.environ['DOCKER_SERVER_API_VERSION']) container = client.create_container(image="debian:wheezy", stdin_open=True, tty=True, command='echo "passed"') dockerpty.start(client, container)
