def __init__(self,
root,
datmo_directory_name,
docker_execpath="docker",
docker_socket=None):
super(DockerEnvironmentDriver, self).__init__()
if not docker_socket:
if platform.system() != "Windows":
docker_socket = "unix:///var/run/docker.sock"
self.root = root
# Check if filepath exists
if not os.path.exists(self.root):
raise PathDoesNotExist(
__("error",
"controller.environment.driver.docker.__init__.dne", root))
self._datmo_directory_name = datmo_directory_name
self._datmo_directory_path = os.path.join(self.root,
self._datmo_directory_name)
self.environment_directory_name = "environment"
self.environment_directory_path = os.path.join(
self._datmo_directory_path, self.environment_directory_name)
self.docker_execpath = docker_execpath
self.docker_socket = docker_socket
if self.docker_socket:
self.client = DockerClient(base_url=self.docker_socket)
self.prefix = [self.docker_execpath, "-H", self.docker_socket]
else:
self.client = DockerClient()
self.prefix = [self.docker_execpath]
self._is_connected = False
self._is_initialized = self.is_initialized
self.type = "docker"
with open(docker_config_filepath) as f:
self.docker_config = json.load(f)
def container(request):
if request.session.get('is_login', None):
if request.method == 'GET':
page = request.GET.get('page')
rows = request.GET.get('limit')
hostid = request.GET.get('hostid')
containerid = request.GET.get('containerid')
ip = EwsHost.objects.get(pk=hostid).ip
if containerid:
try:
client = DockerClient(base_url='tcp://' + ip + ':2375')
containerinfo = client.containers.get(containerid).attrs
return HttpResponse(json.dumps(containerinfo))
except Exception as ex:
return HttpResponse(json.dumps({"API调用异常"}))
elif not containerid:
i = (int(page) - 1) * int(rows)
j = (int(page) - 1) * int(rows) + int(rows)
# 根据ip，调用docker engine api获取容器
client = DockerClient(base_url='tcp://' + ip + ':2375')
containers = client.containers.list(all=True)
total = len(containers)
containers = containers[i:j]
resultdict = {}
dict = []
for cont in containers:
dic = {}
dic['short_id'] = cont.short_id
dic['name'] = cont.name
dic['status'] = cont.status
dict.append(dic)
resultdict['code'] = 0
resultdict['msg'] = ""
resultdict['count'] = total
resultdict['data'] = dict
return JsonResponse(resultdict, safe=False)
if request.method == 'DELETE':
containerid = QueryDict(request.body).get('containerid')
hostid = QueryDict(request.body).get('hostid')
ip = EwsHost.objects.get(pk=hostid).ip
if (not containerid) or (not ip):
return HttpResponse(
json.dumps({
"status": 2,
"msg": "缺少变量containerid和hostid"
}))
try:
client = DockerClient(base_url='tcp://' + ip + ':2375')
containerins = client.containers.get(containerid)
containerins.remove(v=True, force=True)
return HttpResponse(json.dumps({"status": 0}))
except Exception as ex:
return HttpResponse(json.dumps({
"status": 1,
"msg": "API调用异常"
}))
def image(request):
if request.session.get('is_login', None):
if request.method == 'GET':
page = request.GET.get('page')
rows = request.GET.get('limit')
id = request.GET.get('hostid')
imageid = request.GET.get('imageid')
ip = EwsHost.objects.get(pk=id).ip
if imageid:
try:
client = DockerClient(base_url='tcp://' + ip + ':2375')
imageinfo = client.images.get(imageid).attrs
return HttpResponse(json.dumps(imageinfo))
except Exception as ex:
return HttpResponse(json.dumps({"API调用异常"}))
elif not imageid:
i = (int(page) - 1) * int(rows)
j = (int(page) - 1) * int(rows) + int(rows)
# 根据ip，调用docker engine api获取镜像
client = DockerClient(base_url='tcp://' + ip + ':2375')
images = client.images.list()
total = len(images)
images = images[i:j]
resultdict = {}
dict = []
for img in images:
dic = {}
dic['short_id'] = img.short_id
dic['repotag'] = img.attrs.get('RepoTags')
dict.append(dic)
resultdict['code'] = 0
resultdict['msg'] = ""
resultdict['count'] = total
resultdict['data'] = dict
return JsonResponse(resultdict, safe=False)
if request.method == 'DELETE':
imageid = QueryDict(request.body).get('imageid')
hostid = QueryDict(request.body).get('hostid')
ip = EwsHost.objects.get(pk=hostid).ip
if (not imageid) or (not ip):
return HttpResponse(
json.dumps({
"status": 2,
"msg": "缺少变量imageid和hostid"
}))
try:
client = DockerClient(base_url='tcp://' + ip + ':2375')
client.images.remove(imageid, force=True)
return HttpResponse(json.dumps({"status": 0}))
except Exception as ex:
return HttpResponse(json.dumps({
"status": 1,
"msg": "API调用异常"
}))
def deploy_worker(name: str = "", prebuilt: str = '') -> None:
"""
Deploy a new Docker container to a new Docker Machine instance.
:param name: Name of the new machine. If not provided, a name in the format redbot-n will be used, where n is the
current number of machines known to Redbot.
:param prebuilt: Whether the container should first be built locally,
then shipped to the other machines, instead of building it on each machine.
"""
config = {
'vcenter':
get_core_setting('vcenter_host'),
'username':
get_core_setting('vcenter_user'),
'password':
get_core_setting('vcenter_password'),
'network':
get_core_setting('vcenter_mgmt_network'),
'network':
get_core_setting('vcenter_attack_network'),
'hostsystem':
get_core_setting('vcenter_deploy_host'),
'pool':
get_core_setting('vcenter_pool'),
'folder':
get_core_setting('vcenter_folder'),
'datastore':
random.choice(get_core_setting('vcenter_datacenter').split(','))
}
print("Deploy with config", config)
build_mode = get_core_setting('build_mode')
file, image = None, None
# Prepare Container Image
if prebuilt:
c = DockerClient()
image = c.images.get(prebuilt)
elif build_mode == 'local':
c = DockerClient()
image = deploy_container(c)
elif build_mode == 'virtualbox':
m, c = deploy_docker_machine(
'redbot-' + str(storage.scard('machines')), 'virtualbox')
image = deploy_container(c)
if image:
file = image.save()
# Deploy built image to target
m, c = deploy_docker_machine(name
or 'redbot-' + str(storage.scard('machines')),
config=config)
image = deploy_container(c, file)
images.append(image)
class Config():
"""Config data for this particular system.
This currently includes the location of the local docker client and the
local kernel version.
"""
if not uname().sysname == 'Linux':
raise (NotImplementedError(
f"Your system, {uname().sysname}, has not been implemented yet"))
dc = DockerClient('unix://run/docker.sock', version='1.30')
network_name = "docker_application_network"
try:
application_network = dc.networks.list(names=[network_name])[0]
except IndexError:
application_network = dc.networks.create(network_name)
kernel_version_match = find_pattern('\d\.\d\d?', uname().release)
kernel_version = uname().release[kernel_version_match.start(
):kernel_version_match.end()]
kernels = ['2.6', '3.10', '3.13', '3.16', '4.4', '4.9', '4.14', '4.15']
kernel_index = kernels.index(kernel_version)
dockerfile_template = getpath(sep, "usr", "share", "docker-gui",
"Dockerfile.pytemplate")
runscript_template = getpath(sep, "usr", "share", "docker-gui",
"runscript.pytemplate")
user = 1000
group = 1000
user = 1000
group = 1000
def getContainerStatus(container_id):
try:
result = DockerClient(base_url='tcp://127.0.0.1:2375').containers.get(
container_id).status
except NotFound as e:
result = "容器运行失败"
return result
def __init__(self,
master_url: str,
base_url=None,
image: str = 'python:3.7-alpine'):
super().__init__(master_url)
self.docker = DockerClient(base_url=base_url)
self.image = image
def StartContainer():
suri_src_dir = os.path.split(os.path.dirname(
os.path.realpath(__file__)))[0]
print "Using base src dir: " + suri_src_dir
if GOT_DOCKERPY_API < 2:
cli = Client()
cli.start(
'suri-buildbot',
port_bindings={
8010: 8010,
22: None
},
binds={
suri_src_dir: {
'bind': '/data/oisf',
'ro': True
},
os.path.join(suri_src_dir, 'qa', 'docker', 'buildbot.cfg'): {
'bind': '/data/buildbot/master/master.cfg',
'ro': True
}
})
else:
cli = DockerClient()
cli.containers.get('suri-buildbot').start()
sys.exit(0)
def __init__(self,
context,
spec,
build_status=None,
docker_version='auto'):
self.context = context
self.spec = spec
self.repo_name = context.repository.split('/')[-1]
self.commit_hash = context.source['commit']['hash']
self.build_status = build_status or BuildStatus(
bitbucket, context.source['repository']['full_name'],
self.commit_hash, 'badwolf/test',
url_for('log.build_log', sha=self.commit_hash, _external=True))
self.docker = DockerClient(
base_url=current_app.config['DOCKER_HOST'],
timeout=current_app.config['DOCKER_API_TIMEOUT'],
version=docker_version,
)
vault_url = spec.vault.url or current_app.config['VAULT_URL']
vault_token = spec.vault.token or current_app.config['VAULT_TOKEN']
if vault_url and vault_token:
self.vault = hvac.Client(url=vault_url, token=vault_token)
else:
self.vault = None
def __init__(
self,
tag,
base_url='unix://var/run/docker.sock',
container_recipe='/tmp',
container_conda_bld='/home/{username}/conda-bld',
host_conda_bld=None,
image='condaforge/linux-anvil',
verbose=False,
):
"""
Builds a container based on `image`, adding the local user and group to
the container.
"""
self.tag = tag
self.image = image
self.host_conda_bld = host_conda_bld
self.verbose = verbose
uid = os.getuid()
usr = pwd.getpwuid(uid)
self.user_info = dict(uid=uid,
gid=usr.pw_gid,
groupname=grp.getgrgid(usr.pw_gid).gr_name,
username=usr.pw_name)
self.container_recipe = container_recipe
self.container_conda_bld = container_conda_bld.format(**self.user_info)
self.docker = DockerClient(base_url=base_url)
self._build = None
def _check_docker_image_exists(self, image_name):
"""
Query the docker service and check if the given image exists
:param image_name: name of the docker image
:return:
"""
return len(DockerClient().images.list(name=image_name)) > 0
def container_logs(request):
if request.session.get('is_login', None):
if request.method == 'POST':
containerid = request.POST.get('containerid')
hostid = request.POST.get('hostid')
ip = EwsHost.objects.get(pk=hostid).ip
if (not containerid) or (not ip):
return HttpResponse(
json.dumps({
"status": 2,
"msg": "缺少变量containerid和hostid"
}))
try:
client = DockerClient(base_url='tcp://' + ip + ':2375')
containerins = client.containers.get(containerid)
str_out = containerins.logs(tail=200).decode()
return HttpResponse(
json.dumps({
"status": 0,
"msg": "成功",
"data": str_out
}))
except Exception as ex:
print(ex)
return HttpResponse(json.dumps({
"status": 1,
"msg": "API调用异常"
}))
def run(self):
client = DockerClient(base_url='unix://var/run/docker.sock')
source = self.command['src']
credentials = self.command['credentials']
# Start 'lofar-stage2' container
container = client.containers.run(image='lofar-stage2',
auto_remove=True,
detach=True,
tty=True,
network='lofar-net',
environment={
'LOFAR_USER':
credentials['lofarUsername'],
'LOFAR_PASS':
credentials['lofarPassword']
})
# Reload container information
sleep(1)
container.reload()
# bootstrap
hostname = container.attrs['Config']['Hostname']
response = post(url=f'http://{hostname}:5000/stage',
json={
'id': self.identifier,
'cmd': self.command,
'webhook': self.webhook,
'options': {}
})
return loads(response.text)
def build_slave(settings, **kwargs) -> bool:
print("Setting up connection to local docker client")
docker = DockerClient()
print("Building image...")
image = docker.images.build(
tag="jenkins-slave",
path="jenkins-slave",
rm=True,
)
versioned_tag = SLAVE_IMAGE + ":latest"
remote_tag = "{0}/{1}/{2}:latest".format(
settings.repository_url,
settings.repository_group,
SLAVE_IMAGE,
)
print("Tagging image")
image.tag(versioned_tag)
image.tag(remote_tag)
print("Pushing image {0}...".format(remote_tag))
docker.images.push(repository=remote_tag,
auth_config={
"username": settings.repository_user,
"password": settings.repository_password,
})
print("Successfully pushed image to repository")
return True
def __init__(self):
from docker import DockerClient
self.client = DockerClient(
base_url=config.DOCKER_BASE_URL,
timeout=config.DOCKER_TIMEOUT,
num_pools=config.DOCKER_NUM_POOLS
)
def __init__(self, client):
self.client = client
self.dclient = DockerClient(**self.client._connect_params)
self.dclient.api = client
self.parameters = TaskParameters(client)
self.check_mode = self.client.check_mode
self.results = {
u'changed': False,
u'actions': []
}
self.diff = self.client.module._diff
self.diff_tracker = DifferenceTracker()
self.diff_result = dict()
self.existing_plugin = self.get_existing_plugin()
state = self.parameters.state
if state == 'present':
self.present()
elif state == 'absent':
self.absent()
elif state == 'enable':
self.enable()
elif state == 'disable':
self.disable()
if self.diff or self.check_mode or self.parameters.debug:
if self.diff:
self.diff_result['before'], self.diff_result['after'] = self.diff_tracker.get_before_after()
self.results['diff'] = self.diff_result
def web_driver_docker_client(self) -> Optional[DockerClient]:
if not self.ssh_login_info:
return None
SSHAgent.add_keys((self.ssh_login_info["key_file"], ))
# since a bug in docker package https://github.com/docker-library/python/issues/517 that need to explicitly
# pass down the port for supporting ipv6
user = self.ssh_login_info['user']
hostname = normalize_ipv6_url(self.ssh_login_info['hostname'])
try:
return DockerClient(base_url=f"ssh://{user}@{hostname}:22", timeout=DOCKER_API_CALL_TIMEOUT)
except paramiko.ssh_exception.BadHostKeyException as exc:
system_host_keys_path = os.path.expanduser("~/.ssh/known_hosts")
system_host_keys = paramiko.hostkeys.HostKeys(system_host_keys_path)
if system_host_keys.pop(exc.hostname, None):
system_host_keys.save(system_host_keys_path)
return DockerClient(base_url=f"ssh://{user}@{hostname}:22", timeout=DOCKER_API_CALL_TIMEOUT)
def execute(self, callFunc):
import paramiko
from docker import DockerClient
from sshtunnel import SSHTunnelForwarder
import time
for host in self.adress:
user = host.split('@', 1)[0]
addr = host.split('@', 1)[1]
port = self.localConnectionPort
forward = SSHTunnelForwarder(
(addr, 22),
ssh_username=user,
ssh_pkey="/var/ssh/rsa_key",
ssh_private_key_password="******",
remote_bind_address=(addr, 2375),
local_bind_address=('127.0.0.1', port))
forward.start()
time.sleep(3)
dockerConection = DockerClient(
'tcp://127.0.0.1:{port}'.format(port=port))
res = callFunc(dockerConection)
dockerConection.close()
del dockerConection
forward.stop()
return res
def registry(tmpdir_factory, pytestconfig):
if not has_docker() or 'not docker' in pytestconfig.getoption('markexpr'):
pytest.skip('skipping docker tests')
with Container('registry:latest').with_exposed_ports(
REGISTRY_PORT) as container:
host = f'localhost:{container.get_exposed_port(REGISTRY_PORT)}'
client = DockerClient()
# migrate our image to custom Docker registry
tag_name = f'{host}/{REPOSITORY_NAME}/{IMAGE_NAME}'
client.images.pull(IMAGE_NAME, 'latest').tag(tag_name)
client.images.push(tag_name)
tmpdir = str(tmpdir_factory.mktemp("image"))
# create failing image: alpine is too small to have python inside
with open(os.path.join(tmpdir, 'Dockerfile'), 'w') as f:
f.write("""
FROM alpine:latest
CMD python
""")
broken_tag_name = f'{host}/{REPOSITORY_NAME}/{BROKEN_IMAGE_NAME}'
client.images.build(path=tmpdir, tag=broken_tag_name)
client.images.push(broken_tag_name)
yield RemoteDockerRegistry(host)
def __init__(self, base_master_name, base_slave_name, base_tag,
tag_decoration):
super(TagBareImageProvider, self).__init__(tag_decoration)
self.base_master_name = base_master_name
self.base_slave_name = base_slave_name
self.base_tag = base_tag
self.client = DockerClient()
def docker():
client = DockerClient("unix:///var/run/docker.sock")
already_swarm = client.info()["Swarm"]["LocalNodeState"] == "active"
if not already_swarm:
client.swarm.init()
network = client.networks.create("dockerspawner-test-network", driver="overlay", attachable=True)
network.connect("dockerspawner-test")
try:
yield client
finally:
for service in client.services.list():
if service.name.startswith("jupyterhub-client"):
service.scale(0)
for _ in range(10):
if not service.tasks():
break
sleep(1)
service.remove()
network.disconnect("dockerspawner-test")
network.remove()
if not already_swarm:
client.swarm.leave(True)
def StopContainer():
if GOT_DOCKERPY_API < 2:
cli = Client()
cli.stop('suri-buildbot')
else:
cli = DockerClient()
cli.containers.get('suri-buildbot').stop()
sys.exit(0)
def web_driver_docker_client(self) -> Optional[DockerClient]:
if not self.ssh_login_info:
return None
SSHAgent.add_keys((self.ssh_login_info["key_file"], ))
return DockerClient(
base_url=
f"ssh://{self.ssh_login_info['user']}@{self.ssh_login_info['hostname']}",
timeout=DOCKER_API_CALL_TIMEOUT)
def docker_client():
# todo improve when yellowbox is upgraded
try:
ret = DockerClient.from_env()
ret.ping()
except Exception:
return DockerClient(base_url="tcp://localhost:2375")
else:
return ret
def removeContainerFromDockerFile(topic_id: int):
client = DockerClient(base_url='tcp://127.0.0.1:2375')
instances = TopicInstance.objects.filter(topic_id=topic_id)
for instance in instances:
try:
client.containers.get(instance.container_id).remove(force=True)
except NotFound as e:
print("容器不存在")
instance.delete()
def dock_upload_image(tag):
status = True
dc = DockerClient(base_url=CONF.clients_docker.url)
resp = [line for line in dc.push(tag, stream=True)]
for l in resp:
if "error" in l.lower():
status = False
LOG.debug(l.strip())
return status
def _create_wrapper():
""" Creates the docker client wrapper"""
from docker import DockerClient
from wrapper import DockerWrapper
from models import RunnerConfig
from auth.authfactory import AuthenticationFactory
socket_path = os.environ.get('DOCKER_SOCKET_PATH', 'unix://var/run/docker.sock')
return DockerWrapper(DockerClient(base_url=socket_path), RunnerConfig.from_environ(), AuthenticationFactory())
def get_docker_client(machine_name: str) -> DockerClient:
"""
Given the name of a Docker Machine instance, return an instance of DockerClient that is configured to communicate
with the machine.
:param machine_name: Docker Machine instance name to interact with.
:return: DockerClient instance configured for the machine.
"""
m = Machine()
return DockerClient(**m.config(machine=machine_name))
def __init__(self, option):
self.docker = DockerClient()
if option.list:
data = self.containers()
elif option.host:
data = self.containers_by_host(option.host)
else:
data = self._data_structure
print(dumps(data))
def docker_client(self):
if self._docker_client is None:
kwargs = kwargs_from_env()
try:
kwargs['tls'].assert_hostname = False
except KeyError:
pass
kwargs['version'] = 'auto'
self._docker_client = DockerClient(**kwargs)
return self._docker_client
