def test_custom_user_agent(self):
client = APIClient(user_agent='foo/bar')
client.version()
self.assertEqual(self.mock_send.call_count, 1)
headers = self.mock_send.call_args[0][0].headers
self.assertEqual(headers['User-Agent'], 'foo/bar')
def test_custom_user_agent(self):
client = APIClient(user_agent='foo/bar')
client.version()
assert self.mock_send.call_count == 1
headers = self.mock_send.call_args[0][0].headers
assert headers['User-Agent'] == 'foo/bar'
class BaseAPIClientTest(unittest.TestCase):
def setUp(self):
self.patcher = mock.patch.multiple(
'docker.api.client.APIClient',
get=fake_get,
post=fake_post,
put=fake_put,
delete=fake_delete,
_read_from_socket=fake_read_from_socket
)
self.patcher.start()
self.client = APIClient(version=DEFAULT_DOCKER_API_VERSION)
def tearDown(self):
self.client.close()
self.patcher.stop()
def base_create_payload(self, img='busybox', cmd=None):
if not cmd:
cmd = ['true']
return {"Tty": False, "Image": img, "Cmd": cmd,
"AttachStdin": False,
"AttachStderr": True, "AttachStdout": True,
"StdinOnce": False,
"OpenStdin": False, "NetworkDisabled": False,
}
class BaseAPIClientTest(unittest.TestCase):
def setUp(self):
self.patcher = mock.patch.multiple(
'docker.api.client.APIClient',
get=fake_get,
post=fake_post,
put=fake_put,
delete=fake_delete,
_read_from_socket=fake_read_from_socket
)
self.patcher.start()
self.client = APIClient()
# Force-clear authconfig to avoid tampering with the tests
self.client._cfg = {'Configs': {}}
def tearDown(self):
self.client.close()
self.patcher.stop()
def base_create_payload(self, img='busybox', cmd=None):
if not cmd:
cmd = ['true']
return {"Tty": False, "Image": img, "Cmd": cmd,
"AttachStdin": False,
"AttachStderr": True, "AttachStdout": True,
"StdinOnce": False,
"OpenStdin": False, "NetworkDisabled": False,
}
def test_default_user_agent(self):
client = APIClient()
client.version()
self.assertEqual(self.mock_send.call_count, 1)
headers = self.mock_send.call_args[0][0].headers
expected = 'docker-sdk-python/%s' % docker.__version__
self.assertEqual(headers['User-Agent'], expected)
def test_default_user_agent(self):
client = APIClient()
client.version()
assert self.mock_send.call_count == 1
headers = self.mock_send.call_args[0][0].headers
expected = 'docker-sdk-python/%s' % docker.__version__
assert headers['User-Agent'] == expected
def setUp(self):
self.patcher = mock.patch.multiple(
'docker.api.client.APIClient',
get=fake_get,
post=fake_post,
put=fake_put,
delete=fake_delete,
_read_from_socket=fake_read_from_socket)
self.patcher.start()
self.client = APIClient(version=DEFAULT_DOCKER_API_VERSION)
def setUp(self):
self.patcher = mock.patch.multiple(
'docker.api.client.APIClient',
get=fake_get,
post=fake_post,
put=fake_put,
delete=fake_delete,
_read_from_socket=fake_read_from_socket)
self.patcher.start()
self.client = APIClient()
# Force-clear authconfig to avoid tampering with the tests
self.client._cfg = {'Configs': {}}
def execute(self, context):
logging.info('Starting docker container from image ' + self.image)
tls_config = None
if self.tls_ca_cert and self.tls_client_cert and self.tls_client_key:
tls_config = tls.TLSConfig(ca_cert=self.tls_ca_cert,
client_cert=(self.tls_client_cert,
self.tls_client_key),
verify=True,
ssl_version=self.tls_ssl_version,
assert_hostname=self.tls_hostname)
self.docker_url = self.docker_url.replace('tcp://', 'https://')
self.cli = DockerAPIClient(base_url=self.docker_url,
version=self.api_version,
tls=tls_config)
if ':' not in self.image:
image = self.image + ':latest'
else:
image = self.image
if self.force_pull or len(self.cli.images(name=image)) == 0:
logging.info('Pulling docker image ' + image)
for l in self.cli.pull(image, stream=True):
output = json.loads(l.decode('utf-8'))
logging.info("{}".format(output['status']))
cpu_shares = int(round(self.cpus * 1024))
with TemporaryDirectory(prefix='airflowtmp') as host_tmp_dir:
self.environment['AIRFLOW_TMP_DIR'] = self.tmp_dir
self.volumes.append('{0}:{1}'.format(host_tmp_dir, self.tmp_dir))
self.container = self.cli.create_container(
command=self.get_command(),
cpu_shares=cpu_shares,
environment=self.environment,
host_config=self.cli.create_host_config(
binds=self.volumes, network_mode=self.network_mode),
image=image,
mem_limit=self.mem_limit,
user=self.user)
self.cli.start(self.container['Id'])
line = ''
for line in self.cli.logs(container=self.container['Id'],
stream=True):
line = line.strip()
if hasattr(line, 'decode'):
line = line.decode('utf-8')
logging.info(line)
exit_code = self.cli.wait(self.container['Id'])
if exit_code != 0:
raise AirflowException('docker container failed')
if self.xcom_push_flag:
return self.cli.logs(container=self.container['Id']
) if self.xcom_all else str(line)
def request(self, stream=None, tty=None, demux=None):
assert stream is not None and tty is not None and demux is not None
with APIClient(base_url=self.address) as client:
if tty:
url = client._url('/tty')
else:
url = client._url('/no-tty')
resp = client._post(url, stream=True)
return client._read_from_socket(
resp, stream=stream, tty=tty, demux=demux)
def setUp(self):
self.patcher = mock.patch.multiple(
'docker.api.client.APIClient',
get=fake_get,
post=fake_post,
put=fake_put,
delete=fake_delete,
_read_from_socket=fake_read_from_socket
)
self.patcher.start()
self.client = APIClient()
def setUp(self):
self.patcher = mock.patch.multiple(
'docker.api.client.APIClient',
get=fake_get,
post=fake_post,
put=fake_put,
delete=fake_delete,
_read_from_socket=fake_read_from_socket
)
self.patcher.start()
self.client = APIClient()
# Force-clear authconfig to avoid tampering with the tests
self.client._cfg = {'Configs': {}}
class DisableSocketTest(unittest.TestCase):
class DummySocket(object):
def __init__(self, timeout=60):
self.timeout = timeout
def settimeout(self, timeout):
self.timeout = timeout
def gettimeout(self):
return self.timeout
def setUp(self):
self.client = APIClient()
def test_disable_socket_timeout(self):
"""Test that the timeout is disabled on a generic socket object."""
socket = self.DummySocket()
self.client._disable_socket_timeout(socket)
assert socket.timeout is None
def test_disable_socket_timeout2(self):
"""Test that the timeouts are disabled on a generic socket object
and it's _sock object if present."""
socket = self.DummySocket()
socket._sock = self.DummySocket()
self.client._disable_socket_timeout(socket)
assert socket.timeout is None
assert socket._sock.timeout is None
def test_disable_socket_timout_non_blocking(self):
"""Test that a non-blocking socket does not get set to blocking."""
socket = self.DummySocket()
socket._sock = self.DummySocket(0.0)
self.client._disable_socket_timeout(socket)
assert socket.timeout is None
assert socket._sock.timeout == 0.0
class DockerOperator(BaseOperator):
"""
Execute a command inside a docker container.
A temporary directory is created on the host and mounted into a container to allow storing files
that together exceed the default disk size of 10GB in a container. The path to the mounted
directory can be accessed via the environment variable ``AIRFLOW_TMP_DIR``.
:param image: Docker image from which to create the container.
:type image: str
:param api_version: Remote API version.
:type api_version: str
:param command: Command to be run in the container.
:type command: str or list
:param cpus: Number of CPUs to assign to the container.
This value gets multiplied with 1024. See
https://docs.docker.com/engine/reference/run/#cpu-share-constraint
:type cpus: float
:param docker_url: URL of the host running the docker daemon.
:type docker_url: str
:param environment: Environment variables to set in the container.
:type environment: dict
:param force_pull: Pull the docker image on every run.
:type force_pull: bool
:param mem_limit: Maximum amount of memory the container can use. Either a float value, which
represents the limit in bytes, or a string like ``128m`` or ``1g``.
:type mem_limit: float or str
:param network_mode: Network mode for the container.
:type network_mode: str
:param tls_ca_cert: Path to a PEM-encoded certificate authority to secure the docker connection.
:type tls_ca_cert: str
:param tls_client_cert: Path to the PEM-encoded certificate used to authenticate docker client.
:type tls_client_cert: str
:param tls_client_key: Path to the PEM-encoded key used to authenticate docker client.
:type tls_client_key: str
:param tls_hostname: Hostname to match against the docker server certificate or False to
disable the check.
:type tls_hostname: str or bool
:param tls_ssl_version: Version of SSL to use when communicating with docker daemon.
:type tls_ssl_version: str
:param tmp_dir: Mount point inside the container to a temporary directory created on the host by
the operator. The path is also made available via the environment variable
``AIRFLOW_TMP_DIR`` inside the container.
:type tmp_dir: str
:param user: Default user inside the docker container.
:type user: int or str
:param volumes: List of volumes to mount into the container, e.g.
``['/host/path:/container/path', '/host/path2:/container/path2:ro']``.
:param xcom_push: Does the stdout will be pushed to the next step using XCom.
The default is False.
:type xcom_push: bool
:param xcom_all: Push all the stdout or just the last line. The default is False (last line).
:type xcom_all: bool
"""
template_fields = ('command', )
template_ext = (
'.sh',
'.bash',
)
@apply_defaults
def __init__(self,
image,
api_version=None,
command=None,
cpus=1.0,
docker_url='unix://var/run/docker.sock',
environment=None,
force_pull=False,
mem_limit=None,
network_mode=None,
tls_ca_cert=None,
tls_client_cert=None,
tls_client_key=None,
tls_hostname=None,
tls_ssl_version=None,
tmp_dir='/tmp/airflow',
user=None,
volumes=None,
xcom_push=False,
xcom_all=False,
*args,
**kwargs):
super(DockerOperator, self).__init__(*args, **kwargs)
self.api_version = api_version
self.command = command
self.cpus = cpus
self.docker_url = docker_url
self.environment = environment or {}
self.force_pull = force_pull
self.image = image
self.mem_limit = mem_limit
self.network_mode = network_mode
self.tls_ca_cert = tls_ca_cert
self.tls_client_cert = tls_client_cert
self.tls_client_key = tls_client_key
self.tls_hostname = tls_hostname
self.tls_ssl_version = tls_ssl_version
self.tmp_dir = tmp_dir
self.user = user
self.volumes = volumes or []
self.xcom_push_flag = xcom_push
self.xcom_all = xcom_all
self.cli = None
self.container = None
def execute(self, context):
logging.info('Starting docker container from image ' + self.image)
tls_config = None
if self.tls_ca_cert and self.tls_client_cert and self.tls_client_key:
tls_config = tls.TLSConfig(ca_cert=self.tls_ca_cert,
client_cert=(self.tls_client_cert,
self.tls_client_key),
verify=True,
ssl_version=self.tls_ssl_version,
assert_hostname=self.tls_hostname)
self.docker_url = self.docker_url.replace('tcp://', 'https://')
self.cli = DockerAPIClient(base_url=self.docker_url,
version=self.api_version,
tls=tls_config)
if ':' not in self.image:
image = self.image + ':latest'
else:
image = self.image
if self.force_pull or len(self.cli.images(name=image)) == 0:
logging.info('Pulling docker image ' + image)
for l in self.cli.pull(image, stream=True):
output = json.loads(l.decode('utf-8'))
logging.info("{}".format(output['status']))
cpu_shares = int(round(self.cpus * 1024))
with TemporaryDirectory(prefix='airflowtmp') as host_tmp_dir:
self.environment['AIRFLOW_TMP_DIR'] = self.tmp_dir
self.volumes.append('{0}:{1}'.format(host_tmp_dir, self.tmp_dir))
self.container = self.cli.create_container(
command=self.get_command(),
cpu_shares=cpu_shares,
environment=self.environment,
host_config=self.cli.create_host_config(
binds=self.volumes, network_mode=self.network_mode),
image=image,
mem_limit=self.mem_limit,
user=self.user)
self.cli.start(self.container['Id'])
line = ''
for line in self.cli.logs(container=self.container['Id'],
stream=True):
line = line.strip()
if hasattr(line, 'decode'):
line = line.decode('utf-8')
logging.info(line)
exit_code = self.cli.wait(self.container['Id'])
if exit_code != 0:
raise AirflowException('docker container failed')
if self.xcom_push_flag:
return self.cli.logs(container=self.container['Id']
) if self.xcom_all else str(line)
def get_command(self):
if self.command is not None and self.command.strip().find('[') == 0:
commands = ast.literal_eval(self.command)
else:
commands = self.command
return commands
def on_kill(self):
if self.cli is not None:
logging.info('Stopping docker container')
self.cli.stop(self.container['Id'])
def test_url_compatibility_http_unix_triple_slash(self):
c = APIClient(
base_url="http+unix:///socket",
version=DEFAULT_DOCKER_API_VERSION)
assert self._socket_path_for_client_session(c) == '/socket'
def setUp(self):
self.client = APIClient()
def setUp(self):
self.client = APIClient(version=DEFAULT_DOCKER_API_VERSION)
def test_url_compatibility_tcp(self):
c = APIClient(
base_url="tcp://hostname:1234",
version=DEFAULT_DOCKER_API_VERSION)
assert c.base_url == "http://hostname:1234"
def test_retrieve_server_version(self):
client = APIClient(version="auto")
assert isinstance(client._version, six.string_types)
assert not (client._version == "auto")
client.close()
class DockerClient(object):
"""
This client is a "translator" between docker-py API and
the standard docker command line. We need one because docker-py
does not use the same naming for command names and their parameters.
For example, "docker ps" is named "containers", "-n" parameter
is named "limit", some parameters are not implemented at all, etc.
"""
def __init__(self,
timeout=None,
clear_handler=None,
refresh_handler=None,
logger=None):
"""
Initialize the Docker wrapper.
:param timeout: int
:param clear_handler: callable
:param refresh_handler: callable
:param logger: logger
"""
assert callable(clear_handler)
assert callable(refresh_handler)
self.logger = logger
self.exception = None
self.handlers = {
'attach': (self.attach, 'Attach to a running container.'),
'build': (self.build, ("Build a new image from the source"
" code")),
'clear': (clear_handler, "Clear the window."),
'create': (self.create, 'Create a new container.'),
'exec': (self.execute, ("Run a command in a running"
" container.")),
'help': (self.help, "Help on available commands."),
'pause': (self.pause, "Pause all processes within a container."),
'ps': (self.containers, "List containers."),
'port': (self.port, ("List port mappings for the container, or "
"lookup the public-facing port that is "
"NAT-ed to the private_port.")),
'pull': (self.pull, ("Pull an image or a repository from the "
"registry.")),
'push': (self.push, ("Push an image or a repository to the "
"registry.")),
'images': (self.images, "List images."),
'info': (self.info, "Display system-wide information."),
'inspect':
(self.inspect,
"Return low-level information on a " + "container or image."),
'kill': (self.kill, "Kill one or more running containers"),
'login': (self.login, ("Register or log in to a Docker registry "
"server (defaut "
"\"https://index.docker.io/v1/\").")),
'logs': (self.logs, "Fetch the logs of a container."),
'refresh': (refresh_handler, "Refresh autocompletions."),
'rename': (self.rename, "Rename a container."),
'restart': (self.restart, "Restart a running container."),
'run': (self.run, "Run a command in a new container."),
'rm': (self.rm, "Remove one or more containers."),
'rmi': (self.rmi, "Remove one or more images."),
'search': (self.search, "Search the Docker Hub for images."),
'shell': (self.shell, "Get shell into a running container."),
'start': (self.start, "Restart a stopped container."),
'stop': (self.stop, "Stop a running container."),
'tag': (self.tag, "Tag an image into a repository."),
'top': (self.top, "Display the running processes of a container."),
'unpause': (self.unpause, ("Unpause all processes within a "
"container.")),
'version': (self.version, "Show the Docker version information."),
'volume create': (self.volume_create, "Create a new volume."),
'volume inspect': (self.volume_inspect, "Inspect one or more "
"volumes."),
'volume ls': (self.volume_ls, "List volumes."),
'volume rm': (self.volume_rm, "Remove a volume."),
}
self.output = None
self.after = None
self.command = None
self.log = None
self.is_refresh_containers = False
self.is_refresh_running = False
self.is_refresh_images = False
self.is_refresh_volumes = False
disable_warnings()
if sys.platform.startswith('darwin') \
or sys.platform.startswith('win32'):
try:
# mac or win
kwargs = kwargs_from_env()
# hack from here:
# http://docker-py.readthedocs.org/en/latest/boot2docker/
# See also: https://github.com/docker/docker-py/issues/406
if 'tls' in kwargs:
kwargs['tls'].assert_hostname = False
kwargs['timeout'] = timeout
self.instance = DockerAPIClient(**kwargs)
except DockerException as x:
if 'CERTIFICATE_VERIFY_FAILED' in str(x):
raise DockerSslException(x)
elif 'ConnectTimeoutError' in str(x):
raise DockerTimeoutException(x)
else:
raise x
else:
# unix-based
kwargs = kwargs_from_env(ssl_version=ssl.PROTOCOL_TLSv1,
assert_hostname=False)
kwargs['timeout'] = timeout
self.instance = DockerAPIClient(**kwargs)
def debug(self, message):
"""Log a debug message if logger is passed in."""
if self.logger is not None:
self.logger.debug(message)
def handle_input(self, text):
"""
Parse the command, run it via the client, and return
some iterable output to print out. This will parse options
and arguments out of the command line and into parameters
consistent with docker-py naming. It is designed to be the
only really public method of the client. Other methods
are just pass-through methods that delegate commands
to docker-py.
:param text: user input
:return: iterable
"""
def reset_output():
""" Set all internals to initial state."""
self.command = None
self.is_refresh_containers = False
self.is_refresh_running = False
self.is_refresh_images = False
self.is_refresh_volumes = False
self.after = None
self.log = None
self.exception = None
tokens = shlex_split(text) if text else ['']
cmd, params = split_command_and_args(tokens)
reset_output()
if cmd and cmd in self.handlers:
handler = self.handlers[cmd][0]
self.command = cmd
if params:
try:
if '-h' in tokens or '--help' in tokens:
self.output = [format_command_help(cmd)]
else:
parser, popts, pargs = parse_command_options(
cmd, params)
if 'help' in popts:
del popts['help']
self.output = handler(*pargs, **popts)
except APIError as ex:
reset_output()
self.output = [str(ex.explanation)]
except OptionError as ex:
reset_output()
raise ex
except Exception as ex:
reset_output()
self.output = [ex.__repr__()]
else:
self.output = handler()
elif cmd:
self.output = self.help()
def attach(self, *args, **kwargs):
"""
Attach to a running container.
:param kwargs:
:return: None
"""
if not args:
return ['Container name or ID is required.']
container = args[0]
def on_after():
self.is_refresh_containers = True
self.is_refresh_running = True
return ['\rDetached from {0}.'.format(container)]
self.after = on_after
command = format_command_line('attach', False, args, kwargs)
process = pexpect.spawnu(command)
process.interact()
def help(self, *_):
"""
Collect and return help docstrings for all commands.
:return: list of tuples
"""
help_rows = [(key, self.handlers[key][1]) for key in COMMAND_NAMES]
return help_rows
def not_implemented(self, *_, **kw):
"""
Placeholder for commands to be implemented.
:return: iterable
"""
return ['Not implemented.']
def version(self, *_):
"""
Return the version. Equivalent of docker version.
:return: list of tuples
"""
try:
verdict = self.instance.version()
return verdict
except ConnectionError as ex:
raise DockerPermissionException(ex)
def info(self, *_):
"""
Return the system info. Equivalent of docker info.
:return: list of tuples
"""
info_dict = self.instance.info()
return info_dict
def inspect(self, *args, **_):
"""
Return image or container info. Equivalent of docker inspect.
:return: dict
"""
if not args or len(args) == 0:
yield 'Container or image ID is required.'
cids, cnames, imids, imnames = set(), set(), set(), set()
cs = self.containers(all=True)
if cs and len(cs) > 0 and isinstance(cs[0], dict):
cids = set([c['Id'] for c in cs])
cnames = set([name for c in cs for name in c['Names']])
ims = self.images(all=True)
if ims and len(ims) > 0 and isinstance(ims[0], dict):
imids = set([i['Id'] for i in ims])
imnames = set([i['Repository'] for i in ims])
for name in args:
if name in cids or name in cnames:
info = self.instance.inspect_container(name)
elif name in imids or name in imnames:
info = self.instance.inspect_image(name)
else:
info = 'Container or image ID is required.'
yield info
def containers(self, *_, **kwargs):
"""
Return the list of containers. Equivalent of docker ps.
:return: list of dicts
"""
# Truncate by default.
if 'trunc' in kwargs and kwargs['trunc'] is None:
kwargs['trunc'] = True
def format_names(names):
"""
Container names start with /.
Let's strip this for readability.
"""
if isinstance(names, list):
formatted = []
for name in names:
if isinstance(name, six.string_types):
formatted.append(name.lstrip('/'))
else:
formatted.append(name)
return formatted
return names
csdict = self.instance.containers(**kwargs)
if len(csdict) > 0:
if 'quiet' not in kwargs or not kwargs['quiet']:
for i in range(len(csdict)):
csdict[i]['Names'] = format_names(csdict[i]['Names'])
csdict[i]['Created'] = pretty.date(csdict[i]['Created'])
return csdict
else:
return ['There are no containers to list.']
def pause(self, *args, **kwargs):
"""
Pause all processes in a container. Equivalent of docker pause.
:param kwargs:
:return: Container ID or iterable output.
"""
if not args:
return ['Container name is required.']
kwargs['container'] = args[0]
self.instance.pause(**kwargs)
return [kwargs['container']]
def port(self, *args, **_):
"""
List port mappings for the container. Equivalent of docker port.
:param kwargs:
:return: Container ID or iterable output.
"""
if not args:
return ['Container name is required.']
port_args = [args[0], None]
port_args[1] = args[1] if len(args) > 1 else None
result = self.instance.port(*port_args)
if result:
return result
result = self.instance.inspect_container(port_args[0])
if result:
result = result.get('NetworkSettings', {}).get('Ports', None)
if result:
return result
return ['There are no port mappings for {0}.'.format(args[0])]
def rm(self, *args, **kwargs):
"""
Remove a container. Equivalent of docker rm.
:param kwargs:
:return: Container ID or iterable output.
"""
truncate_output = False
all_stopped = 'all_stopped' in kwargs and kwargs['all_stopped']
all = 'all' in kwargs and kwargs['all']
if all_stopped:
if args and len(args) > 0:
return ['Provide either --all-stopped, or container name(s).']
containers = self.instance.containers(quiet=True,
filters={'status': 'exited'})
if not containers or len(containers) == 0:
return ['There are no stopped containers.']
containers = [c['Id'] for c in containers]
truncate_output = True
elif all:
if args and len(args) > 0:
return ['Provide either --all, or container name(s).']
containers = self.instance.containers(quiet=True, all=True)
if not containers or len(containers) == 0:
return ['There are no containers.']
containers = [c['Id'] for c in containers]
truncate_output = True
else:
containers = args
kwargs = allowed_args('rm', **kwargs)
def stream():
for container in containers:
try:
self.instance.remove_container(container, **kwargs)
self.is_refresh_containers = True
self.is_refresh_running = True
if truncate_output:
yield "{0:.25}".format(container)
else:
yield container
except APIError as ex:
yield '{0:.25}: {1}'.format(container, ex.explanation)
yield 'Removed: {0} container(s).'.format(
len(containers) if containers else 0)
return stream()
def rmi(self, *args, **kwargs):
"""
Remove an image. Equivalent of docker rm.
:param kwargs:
:return: Image name.
"""
truncate_output = False
all_dangling = 'all_dangling' in kwargs and kwargs['all_dangling']
all = 'all' in kwargs and kwargs['all']
if all_dangling:
if args and len(args) > 0:
return ['Provide either --all-dangling, or image name(s).']
images = self.instance.images(quiet=True,
filters={'dangling': True})
if not images or len(images) == 0:
return ['There are no dangling images.']
truncate_output = True
elif all:
if args and len(args) > 0:
return ['Provide either --all, or image name(s).']
images = self.instance.images(quiet=True, all=True)
if not images or len(images) == 0:
return ['There are no images.']
truncate_output = True
else:
images = args
kwargs = allowed_args('rmi', **kwargs)
def stream():
for image in images:
try:
self.instance.remove_image(image, **kwargs)
self.is_refresh_images = True
if truncate_output:
yield "{:.25}".format(image)
else:
yield image
except APIError as ex:
yield '{0:.25}: {1}'.format(image, ex.explanation)
return stream()
def run(self, *args, **kwargs):
"""
Create and start a container. Equivalent of docker run.
:param kwargs:
:return: Container ID or iterable output.
"""
if not args:
return ['Image name is required.']
if kwargs['remove'] and kwargs['detach']:
return ['Use either --rm or --detach.']
# Always call external cli for this, rather than figuring out
# why docker-py throws "jack is incompatible with use of CloseNotifier in same ServeHTTP call"
kwargs['force'] = True
called, args, kwargs = self.call_external_cli('run', *args, **kwargs)
if not called:
kwargs['image'] = args[0]
kwargs['command'] = args[1:] if len(args) > 1 else []
kwargs = self._add_port_bindings(kwargs)
kwargs = self._add_exposed_ports(kwargs)
kwargs = self._add_link_bindings(kwargs)
kwargs = self._add_volumes_from(kwargs)
kwargs = self._add_volumes(kwargs)
kwargs = self._add_network_mode(kwargs)
create_args = allowed_args('create', **kwargs)
result = self.instance.create_container(**create_args)
if result:
if "Warnings" in result and result['Warnings']:
return [result['Warnings']]
if "Id" in result and result['Id']:
self.is_refresh_containers = True
is_attach = 'detach' not in kwargs or not kwargs['detach']
start_args = allowed_args('start', **kwargs)
start_args.update({
'container': result['Id'],
'attach': is_attach
})
return self.start(**start_args)
return ['There was a problem running the container.']
def create(self, *args, **kwargs):
"""
Create a container. Equivalent of docker create.
:param kwargs:
:return: Container ID or iterable output.
"""
if not args:
return ['Image name is required.']
called, args, kwargs = self.call_external_cli('create', *args,
**kwargs)
if not called:
kwargs['image'] = args[0]
kwargs['command'] = args[1:] if len(args) > 1 else []
kwargs = self._add_port_bindings(kwargs)
kwargs = self._add_exposed_ports(kwargs)
kwargs = self._add_link_bindings(kwargs)
kwargs = self._add_volumes_from(kwargs)
kwargs = self._add_volumes(kwargs)
kwargs = self._add_network_mode(kwargs)
create_args = allowed_args('create', **kwargs)
result = self.instance.create_container(**create_args)
if result:
if "Warnings" in result and result['Warnings']:
return [result['Warnings']]
if "Id" in result and result['Id']:
self.is_refresh_containers = True
return [result['Id']]
return ['There was a problem creating the container.']
def rename(self, *args, **kwargs):
"""
Rename a container. Equivalent of docker rename.
:param kwargs:
:return: None.
"""
if not args or len(args) < 2:
return ['Container name and new name are required.']
self.instance.rename(*args)
self.is_refresh_containers = True
def restart(self, *args, **kwargs):
"""
Restart a running container. Equivalent of docker restart.
:param kwargs:
:return: Container ID or iterable output.
"""
if not args:
return ['Container name is required.']
def stream():
for container in args:
self.instance.restart(container, **kwargs)
yield container
return stream()
@if_exception_return(InvalidVersion, None)
def volume_create(self, *args, **kwargs):
"""
Create a volume. Equivalent of docker volume create.
:param kwargs:
:return: Volume name.
"""
if not kwargs:
return ['Volume name is required.']
allowed = allowed_args('volume create', **kwargs)
allowed = self._add_opts(allowed)
result = self.instance.create_volume(**allowed)
self.is_refresh_volumes = True
return [result['Name']]
@if_exception_return(InvalidVersion, None)
def volume_ls(self, *args, **kwargs):
"""
List volumes. Equivalent of docker volume ls.
:param kwargs:
:return: Iterable.
"""
quiet = kwargs.pop('quiet', False)
kwargs = self._add_filters(kwargs)
vdict = self.instance.volumes(**kwargs)
result = vdict.get('Volumes', None)
if result:
if quiet:
result = [volume['Name'] for volume in result]
return result
else:
return ['There are no volumes to list.']
@if_exception_return(InvalidVersion, None)
def volume_rm(self, *args, **kwargs):
"""
Remove a volume. Equivalent of docker volume rm.
:param kwargs:
:return: Volume name.
"""
if not args:
return ['Volume name is required.']
def stream():
for volume in args:
try:
self.instance.remove_volume(volume)
self.is_refresh_volumes = True
yield volume
except APIError as x:
yield 'Could not remove volume {0}: {1}.'.format(
volume, x.explanation)
return stream()
@if_exception_return(InvalidVersion, None)
def volume_inspect(self, *args, **_):
"""
Return volume info. Equivalent of docker volume ls.
:return: dict
"""
if not args or len(args) == 0:
yield 'Volume name is required.'
vnames = self.volume_ls(quiet=True)
for vname in args:
if vname in vnames:
info = self.instance.inspect_volume(vname)
yield info
else:
yield "Volume not found: {0}".format(vname)
def tag(self, *args, **kwargs):
"""
Tag an image into repository. Equivalent of docker tag.
:param kwargs:
:return: Iamge ID.
"""
if not args or len(args) < 2:
return ['Image name and repository name are required.']
img = args[0]
if ':' in args[1]:
repo, tag = args[1].rsplit(':', 1)
else:
repo, tag = args[1], None
result = self.instance.tag(image=img,
repository=repo,
tag=tag,
**kwargs)
if result:
return ['Tagged {0} into {1}.'.format(*args)]
else:
return ['Error tagging {0} into {1}.'.format(*args)]
def _add_filters(self, params):
"""
Update kwargs if filters are present.
:param params: dict
:return dict
"""
if params.get('filters', None):
filters = parse_kv_as_dict(params['filters'], True)
params['filters'] = filters
return params
def _add_opts(self, params):
"""
Update kwargs if opts are present.
:param params: dict
:return dict
"""
if params.get('driver_opts', None):
opts = parse_kv_as_dict(params['driver_opts'], False)
params['driver_opts'] = opts
return params
def _add_volumes(self, params):
"""
Update kwargs if volumes are present.
:param params: dict
:return dict
"""
if params.get('volumes', None):
binds = parse_volume_bindings(params['volumes'])
params['volumes'] = [x['bind'] for x in binds.values()]
conf = self.instance.create_host_config(binds=binds)
self._update_host_config(params, conf)
return params
def _add_volumes_from(self, params):
"""
Update kwargs if volumes-from are present.
:param params: dict
:return dict
"""
if params.get('volumes_from', None):
cs = ','.join(params['volumes_from'])
cs = [x.strip() for x in cs.split(',') if x]
conf = self.instance.create_host_config(volumes_from=cs)
self._update_host_config(params, conf)
return params
def _add_network_mode(self, params):
"""
Update kwargs if --net is present.
:param params: dict
:return: dict
"""
if 'net' in params:
conf = self.instance.create_host_config(network_mode=params['net'])
self._update_host_config(params, conf)
return params
def _add_link_bindings(self, params):
"""
Update kwargs if user wants to link containers.
:param params: dict
:return dict
"""
if params.get('links', None):
links = {}
for link in params['links']:
link_name, link_alias = link.split(':', 1)
links[link_name] = link_alias
link_conf = self.instance.create_host_config(links=links)
self._update_host_config(params, link_conf)
return params
def _add_port_bindings(self, params):
"""
Update kwargs if user wants to bind some ports.
:param params: dict
:return dict
"""
if params.get('port_bindings', None):
port_bindings = parse_port_bindings(params['port_bindings'])
# Have to provide list of ports to open in create_container.
params['ports'] = port_bindings.keys()
# Have to provide host config with port mappings.
port_conf = self.instance.create_host_config(
port_bindings=port_bindings)
self._update_host_config(params, port_conf)
return params
def _add_exposed_ports(self, params):
"""
Update kwargs if user wants to expose some ports.
:param params: dict
:return dict
"""
if params.get('expose', None):
ports = parse_exposed_ports(params['expose'])
# Have to provide list of ports to open in create_container.
params['ports'] = ports.keys()
# Have to provide host config with port mappings.
port_conf = self.instance.create_host_config(port_bindings=ports)
self._update_host_config(params, port_conf)
return params
def _update_host_config(self, params, config_to_merge):
"""
Update config dictionary in kwargs with another dictionary.
:param params: dict
:param config_to_merge: dict with new values
:return dict
"""
if params.get('host_config', None):
params['host_config'].update(config_to_merge)
else:
params['host_config'] = config_to_merge
return params
def _is_repo_tag_valid(self, repo):
"""
When an image is tagged into a repo, make sure only allowed symbols
are used.
:param repo:
:return: (boolean, "error message")
"""
# Username: only [a-z0-9_] are allowed, size between 4 and 30
if '/' not in repo:
return False, 'Format: user_name/repository_name[:tag].'
user_name, repo_name = repo.split('/')
user_pattern = re.compile(r'^[a-z0-9_]{4,30}$')
if not user_pattern.match(user_name):
return False, 'Only [a-z0-9_] are allowed in user name, ' \
'size between 4 and 30'
return True, None
def execute(self, *args, **kwargs):
"""
Execute a command in the container. Equivalent of docker exec.
:param kwargs:
:return: Container ID or iterable output.
"""
if not args or len(args) < 2:
return ['Container ID and command is required.']
called, args, kwargs = self.call_external_cli('exec', *args, **kwargs)
if not called:
kwargs['container'] = args[0]
kwargs['cmd'] = args[1:]
is_detach = kwargs.pop('detach')
exec_args = allowed_args('exec', **kwargs)
result = self.instance.exec_create(**exec_args)
if result and 'Id' in result:
return self.instance.exec_start(result['Id'],
detach=is_detach,
stream=True)
return ['There was a problem executing the command.']
def build(self, *args, **kwargs):
"""
Build an image. Equivalent of docker build.
:param kwargs:
:return: Iterable output.
"""
if not args:
return ['Directory path or URL is required.']
kwargs['path'] = args[0]
kwargs['rm'] = bool(kwargs['rm'])
self.is_refresh_images = True
return self.instance.build(**kwargs)
def shell(self, *args, **_):
"""
Get the shell into a running container. A shortcut for
docker exec -it /usr/bin/env bash.
:param kwargs:
:return: None
"""
if not args:
return ['Container name or ID is required.']
container = args[0]
shellcmd = 'bash'
if len(args) > 1:
shellcmd = ' '.join(args[1:])
self.after = lambda: ['\rShell to {0} is closed.'.format(container)]
command = 'docker exec -it {0} {1}'.format(container, shellcmd)
process = pexpect.spawnu(command)
process.interact()
def start(self, *args, **kwargs):
"""
Start a container. Equivalent of docker start.
:param kwargs:
:return: Container ID or iterable output.
"""
if args:
kwargs['container'] = args[0]
if not kwargs['container']:
return ['Container name is required.']
called, args, kwargs = self.call_external_cli('start', *args, **kwargs)
if not called:
if 'remove' in kwargs and kwargs['remove']:
def on_after():
container = kwargs['container']
try:
self.instance.stop(container)
self.instance.remove_container(container)
yield "Removed container {0:.25} on exit.".format(
container)
except APIError as ex:
yield "{0:.25}: {1}.".format(container, ex.explanation)
self.is_refresh_containers = True
self.is_refresh_running = True
self.after = on_after
startargs = allowed_args('start', **kwargs)
attached = None
if 'attach' in kwargs and kwargs['attach']:
attached = self.view(container=kwargs['container'],
stream=True,
stdout=True,
stderr=False,
logs=False)
result = self.instance.start(**startargs)
# Just in case the stream generated no output, let's allow for
# retrieving the logs. They will be our last resort output.
self.log = lambda: self.instance.logs(kwargs['container'])
self.is_refresh_running = True
if result:
return [result]
elif attached:
return attached
else:
return [kwargs['container']]
def view(self, *_, **kwargs):
"""
Attach to container STDOUT and / or STDERR.
Docker-py does not allow attaching to STDIN.
:param kwargs:
:return: Iterable output
"""
result = self.instance.attach(**kwargs)
return result
def login(self, *args, **kwargs):
"""
Register or log in to a Docker registry server.
:param kwargs:
:return: None
"""
self.after = lambda: ['\r']
command = format_command_line('login', False, args, kwargs)
process = pexpect.spawnu(command)
process.interact()
def logs(self, *args, **kwargs):
"""
Retrieve container logs. Equivalent of docker logs.
:param kwargs:
:return: Iterable output
"""
if not args:
return ['Container ID/name is required.']
kwargs['container'] = args[0]
result = self.instance.logs(**kwargs)
if isinstance(result, bytes):
result = result.decode()
if not kwargs['stream']:
result = [result]
return result
def images(self, *_, **kwargs):
"""
Return the list of images. Equivalent of docker images.
:return: list of dicts
"""
result = self.instance.images(**kwargs)
re_digits = re.compile('^[0-9]+$', re.UNICODE)
def convert_image_dict(a):
"""
Drop some keys and change some values to pretty-print image dict.
"""
b = {}
for k, v in a.items():
if k not in ['RepoTags', 'RepoDigests', 'Labels', 'Size']:
b[k] = v
if k == 'Created' and v and re_digits.search(str(v)):
b[k] = pretty.date(v)
if k == 'VirtualSize':
b[k] = filesize(v)
# If we have more than one repo tag, return as many dicts
if a.get('RepoTags', None) is None:
a['RepoTags'] = ['<none>:<none>']
for rt in a['RepoTags']:
repo, tag = rt.rsplit(':', 1)
c = {}
c.update(b)
c['Repository'] = repo
c['Tag'] = tag
yield c
if len(result) > 0:
if isinstance(result[0], dict):
converted = []
for x in result:
for y in convert_image_dict(x):
converted.append(y)
return converted
return result
else:
return ['There are no images to list.']
def search(self, *args, **_):
"""
Return the list of images matching specified term.
Equivalent of docker search.
:return: list of dicts
"""
if not args or len(args) < 1:
return "Search term is required."
result = self.instance.search(args[0])
if len(result) > 0:
for res in result:
# Make results more readable, like official CLI does.
if 'is_trusted' in res:
res['is_trusted'] = '[OK]' if res['is_trusted'] else ''
if 'is_official' in res:
res['is_official'] = '[OK]' if res['is_official'] else ''
return result
else:
return ['No images were found.']
def stop(self, *args, **kwargs):
"""
Stop a running container. Equivalent of docker stop.
:param kwargs:
:return: Container ID or iterable output.
"""
if not args:
return ['Container name is required.']
def stream():
for container in args:
try:
self.instance.stop(container, **kwargs)
self.is_refresh_running = True
yield container
except APIError as ex:
yield '{0:.25}: {1}'.format(container, ex.explanation)
return stream()
def kill(self, *args, **kwargs):
"""
Kill a running container. Equivalent of docker kill.
:param kwargs:
:return: Container ID or iterable output.
"""
if not args:
return ['Container name is required.']
def stream():
for container in args:
try:
self.instance.kill(container, **kwargs)
self.is_refresh_running = True
yield container
except APIError as ex:
yield '{0:.25}: {1}'.format(container, ex.explanation)
return stream()
def top(self, *args, **kwargs):
"""
Show top processes in a container. Equivalent of docker rm.
:param kwargs:
:return: Container ID or iterable output.
"""
if not args:
return ['Container name is required.']
container = args[0]
result = self.instance.top(container, **kwargs)
return result
def pull(self, *args, **kwargs):
"""
Pull an image by name. Equivalent of docker pull.
:param kwargs:
:return: Container ID or iterable output.
"""
if not args:
return ['Image name is required.']
image = args[0]
kwargs['stream'] = True
result = self.instance.pull(image, **kwargs)
self.is_refresh_images = True
return result
def push(self, *args, **kwargs):
"""
Push an image into repository. Equivalent of docker push.
:param kwargs:
:return: interactive.
"""
if not args or len(args) < 1:
return ['Image name (tagged) is required.']
tag_valid, tag_message = self._is_repo_tag_valid(args[0])
if not tag_valid:
return [tag_message]
self.after = lambda: ['\r']
# TODO: this command didn't have to use pexpect.
# But it was easier to call the official CLI than try and figure out
# why requests throw this error:
# File "venv/wharfee/lib/python2.7/site-packages/requests/packages/
# urllib3/response.py", line 267, in read
# raise ReadTimeoutError(self._pool, None, 'Read timed out.')
# requests.packages.urllib3.exceptions.ReadTimeoutError:
# HTTPSConnectionPool(host='192.168.59.103', port=2376): Read timed out.
command = format_command_line('push', False, args, kwargs)
process = pexpect.spawnu(command)
process.interact()
def unpause(self, *args, **kwargs):
"""
Unpause all processes in a container. Equivalent of docker unpause.
:param kwargs:
:return: Container ID or iterable output.
"""
if not args:
return ['Container name is required.']
kwargs['container'] = args[0]
self.instance.unpause(**kwargs)
return [kwargs['container']]
def call_external_cli(self, cmd, *args, **kwargs):
"""
Call the "official" CLI if needed.
:param args:
:param kwargs:
:return:
"""
called = False
is_force = kwargs.pop('force', False)
is_interactive = kwargs.get('interactive', None)
is_tty = kwargs.get('tty', None)
is_attach = kwargs.get('attach', None)
is_attach_bool = is_attach in [True, False]
def execute_external():
"""
Call the official cli
"""
command = format_command_line(cmd, False, args, kwargs)
process = pexpect.spawnu(command)
process.interact()
def on_after_interactive():
# \r is to make sure when there is some error output,
# prompt is back to beginning of line
self.is_refresh_containers = True
self.is_refresh_running = True
return ['\rInteractive terminal is closed.']
def on_after_attach():
self.is_refresh_containers = True
self.is_refresh_running = True
return ['Container exited.\r']
if is_force or is_interactive or is_tty or (is_attach
and not is_attach_bool):
self.after = on_after_attach if is_attach or (
not is_interactive and not is_tty) else on_after_interactive
called = True
execute_external()
return called, args, kwargs
def __init__(self,
timeout=None,
clear_handler=None,
refresh_handler=None,
logger=None):
"""
Initialize the Docker wrapper.
:param timeout: int
:param clear_handler: callable
:param refresh_handler: callable
:param logger: logger
"""
assert callable(clear_handler)
assert callable(refresh_handler)
self.logger = logger
self.exception = None
self.handlers = {
'attach': (self.attach, 'Attach to a running container.'),
'build': (self.build, ("Build a new image from the source"
" code")),
'clear': (clear_handler, "Clear the window."),
'create': (self.create, 'Create a new container.'),
'exec': (self.execute, ("Run a command in a running"
" container.")),
'help': (self.help, "Help on available commands."),
'pause': (self.pause, "Pause all processes within a container."),
'ps': (self.containers, "List containers."),
'port': (self.port, ("List port mappings for the container, or "
"lookup the public-facing port that is "
"NAT-ed to the private_port.")),
'pull': (self.pull, ("Pull an image or a repository from the "
"registry.")),
'push': (self.push, ("Push an image or a repository to the "
"registry.")),
'images': (self.images, "List images."),
'info': (self.info, "Display system-wide information."),
'inspect':
(self.inspect,
"Return low-level information on a " + "container or image."),
'kill': (self.kill, "Kill one or more running containers"),
'login': (self.login, ("Register or log in to a Docker registry "
"server (defaut "
"\"https://index.docker.io/v1/\").")),
'logs': (self.logs, "Fetch the logs of a container."),
'refresh': (refresh_handler, "Refresh autocompletions."),
'rename': (self.rename, "Rename a container."),
'restart': (self.restart, "Restart a running container."),
'run': (self.run, "Run a command in a new container."),
'rm': (self.rm, "Remove one or more containers."),
'rmi': (self.rmi, "Remove one or more images."),
'search': (self.search, "Search the Docker Hub for images."),
'shell': (self.shell, "Get shell into a running container."),
'start': (self.start, "Restart a stopped container."),
'stop': (self.stop, "Stop a running container."),
'tag': (self.tag, "Tag an image into a repository."),
'top': (self.top, "Display the running processes of a container."),
'unpause': (self.unpause, ("Unpause all processes within a "
"container.")),
'version': (self.version, "Show the Docker version information."),
'volume create': (self.volume_create, "Create a new volume."),
'volume inspect': (self.volume_inspect, "Inspect one or more "
"volumes."),
'volume ls': (self.volume_ls, "List volumes."),
'volume rm': (self.volume_rm, "Remove a volume."),
}
self.output = None
self.after = None
self.command = None
self.log = None
self.is_refresh_containers = False
self.is_refresh_running = False
self.is_refresh_images = False
self.is_refresh_volumes = False
disable_warnings()
if sys.platform.startswith('darwin') \
or sys.platform.startswith('win32'):
try:
# mac or win
kwargs = kwargs_from_env()
# hack from here:
# http://docker-py.readthedocs.org/en/latest/boot2docker/
# See also: https://github.com/docker/docker-py/issues/406
if 'tls' in kwargs:
kwargs['tls'].assert_hostname = False
kwargs['timeout'] = timeout
self.instance = DockerAPIClient(**kwargs)
except DockerException as x:
if 'CERTIFICATE_VERIFY_FAILED' in str(x):
raise DockerSslException(x)
elif 'ConnectTimeoutError' in str(x):
raise DockerTimeoutException(x)
else:
raise x
else:
# unix-based
kwargs = kwargs_from_env(ssl_version=ssl.PROTOCOL_TLSv1,
assert_hostname=False)
kwargs['timeout'] = timeout
self.instance = DockerAPIClient(**kwargs)
def test_retrieve_server_version(self):
client = APIClient(version="auto")
self.assertTrue(isinstance(client._version, six.string_types))
self.assertFalse(client._version == "auto")
client.close()
def test_retrieve_server_version(self):
client = APIClient(version="auto")
self.assertTrue(isinstance(client._version, six.string_types))
self.assertFalse(client._version == "auto")
client.close()
