class docker_monitor: def __init__(self): pass # 查看容器 def display(self, operation): self.host = "tcp://" + operation["host"] + ":2375" self.version = operation["version"] self.client = Client(base_url=self.host) try: if operation["image"] == True: images = self.client.images(all=True) else: images = [] if operation["all_container"] == True: containers = self.client.containers(all=True) else: containers = self.client.containers() result = {"operation": "display", "return": {"images": images, "containers": containers}, "error": None} except: result = {"operation": "display", "return": -1, "error": str(traceback.format_exc())} return result
class DockerManager: """The DockerManager provides a way to manage a Docker engine.""" def __init__(self, docker_url): """docker_url is the URL to the docker engine. It can either be the URL to the docker engine API or a path to the Docker socket. """ self.client = Client(docker_url) def count_containers_by_image(self, container_image): """Iterates through all running containers to count the number of running containers associated with a container image name. Returns the number of running container. """ containers = self.client.containers() service_container_count = 0 for container in containers: if container['Image'] == container_image: service_container_count += 1 return service_container_count def stop_container_by_image(self, container_image): """Iterates through all running containers to stop the first container that match a container image name. """ containers = self.client.containers() for container in containers: if container['Image'] == container_image: self.client.kill(container['Id']) break def create_container(self, service_definition): """Creates a new container based on a service definition object.""" container = self.client.create_container( image=service_definition['image'], command=service_definition['command'], ports=service_definition['ports'], environment=service_definition['environment'], host_config=utils.create_host_config(publish_all_ports=True)) self.client.start(container.get('Id')) def ensure_containers_for_service(self, service_definition, container_number): """Ensures that a number of container are running for a service. Based on the *container_number* parameter it will stop/create containers to match this value. """ service_container_count = self.count_containers_by_image( service_definition['image']) while service_container_count != container_number: if service_container_count > container_number: self.stop_container_by_image(service_definition['image']) service_container_count -= 1 elif service_container_count < container_number: self.create_container(service_definition) service_container_count += 1
def delete_all(): client = Client() for c in client.containers(trunc=False): print("Killing {0}".format(c["Id"])) client.kill(c["Id"]) for c in client.containers(all=True, trunc=False): print("Removing {0}".format(c["Id"])) client.remove_container(c["Id"])
def pcap_queue(path): """ Processes PCAP files that have been added from the rq-worker, and tells vent-management to start plugins for the new file. """ import os import time import ConfigParser config = ConfigParser.RawConfigParser() from docker import Client from docker.utils.types import LogConfig c = Client(base_url='unix://var/run/docker.sock') # first some cleanup containers = c.containers(quiet=True, all=True, filters={'status':"exited"}) for cont in containers: try: # will only remove containers that aren't running c.remove_container(cont['Id']) except Exception as e: pass template_dir = "/var/lib/docker/data/templates/" plugins_dir = "/var/lib/docker/data/plugins/" try: config.read(template_dir+'modes.template') plugin_array = config.options("plugins") plugins = {} for plug in plugin_array: if plug not in ["core", "visualization", "collectors"]: plugins[plug] = config.get("plugins", plug) container_count = 0 container_max = 50 try: config.read(template_dir+'core.template') container_max = int(config.get("active-containers", "count")) except Exception as e: pass for plugin in plugins: # check resources before creating container # wait until there are resources available container_count = len(c.containers(filters={'status':'running'})) while container_count >= container_max: time.sleep(5) container_count = len(c.containers(filters={'status':'running'})) cmd = "python2.7 /vent/template_parser.py "+plugin+" start "+path os.system(cmd) except Exception as e: pass return
def create(imagename): """Create a container from a image. Args: imagename: the image name. """ try: cli = Client(base_url=url) cli.containers() container = cli.create_container(image=imagename, command='/bin/sleep 30') print(container["Id"][:11] + " has been created.") except Exception,e: print str(e)
class DockerAPI(SSHAPI): def __init__(self): try: SSHAPI.__init__(self) config = ConfigParser.ConfigParser() config.read("deploy.cfg") self.docker_client = Client("tcp://%s:2376"%config.get("docker","server")) except Exception as e: print "Falhou ao conectar com Docker:",e def list_Container(self): for c in self.docker_client.containers(all=True): print c.get("Names") def create_container(self,nome,imagem="ubuntu"): res = self.docker_client.create_container(name=nome,image=imagem, command="/bin/bash", tty=True, stdin_open=True, detach=True) if res: return res else: print " Falhou ao criar o container ",res def start_container(self,id): try: self.docker_client.start(container=id) print "Container executado" except Exception as e: print " Falhou ao iniciar o container ",e def get_container(self,nome): try: todos = self.docker_client.containers(all=True) nome = "/"+nome container = [ c for c in todos if nome in c.get("Names") ][0] return container except Exception as e: print " Falhou ao buscar container",e def _exec(self,container,cmd): c = "docker exec %s %s "%(container,cmd) self.exec_ssh_command(c) def get_container_address(self,nome): address = self.get_container(nome) address = address.get("NetworkSettings")\ .get("Networks").get("bridge")\ .get("IPAddress") return address
def __init__(self,host_list=None): if __name__ == "__main__": self.conf = ConfigParser.ConfigParser() self.conf.read("../docker.conf") print self.conf.sections() self.docker_host_list = self.conf.sections() self.docker_base_url = {} for docker_host in self.docker_host_list: self.docker_base_url[docker_host]=self.conf.get(docker_host,"base_url") else: import models self.docker_host_list = models.docker_conf.objects.all() self.docker_base_url = {} for docker_host in self.docker_host_list: try: cli = Client(docker_host.base_url) cli.images() except: # print docker_host.base_url,"is not valied.." continue self.docker_base_url[docker_host.docker_host]=docker_host.base_url # all docker self.all_docker_dic = {} # active docker self.docker_dic={} # pause docker self.pauser_docker_dic = {} # restart docker self.restart_docker_dic = {} # stop docker self.stop_docker_dic = {} self.docker_images = {} self.docker_images_pull = {} for docker_host,base_url in self.docker_base_url.items(): cli = Client(base_url) self.docker_images[docker_host]= cli.images(all=True) self.docker_dic[docker_host]=cli.containers() self.all_docker_dic[docker_host] = cli.containers(all=True) self.pauser_docker_dic[docker_host]= cli.containers(filters={"status":"paused"}) self.restart_docker_dic[docker_host] = cli.containers(filters={"status":"restarting"}) self.stop_docker_dic[docker_host] = cli.containers(filters={"status":"exited"})
def main(): DockerExecutor.set_environment_variables() args = parse_args() final_result_file = args.output origin = GitRepository(args.original_repo, args.original_branch, "original") external = GitRepository(args.external_repo, args.external_branch, "external") tests_to_execute = get_benchmark_list(args.benchmarks) if len(tests_to_execute) == 0: exit(200) if not final_result_file.endswith(".csv"): print("Please specify a CSV file as output") exit(201) # download firstly the image print("Downloading the image " + image + ":" + tag) c = Client(base_url=args.socket_url, version=args.docker_api_version) c.pull(repository=image, tag=tag, insecure_registry=True) # stop all running containers before for container in c.containers(): print(" Stopping container " + container["Id"]) c.remove_container(container["Id"], force=True) print("Done downloading the image") origin_exec = DockerExecutor(origin, args.socket_url, args.docker_api_version) external_exec = DockerExecutor(external, args.socket_url, args.docker_api_version) origin_exec.run(tests_to_execute) parsers_origin = parse_files() shutil.rmtree(tmpdir) # clean up before the next execution os.mkdir(tmpdir) external_exec.run(tests_to_execute) parsers_external = parse_files() Parser.to_csv_multiple_comparison(parsers_origin, parsers_external, final_result_file) print("Done. File " + final_result_file + " created")
def _get_container(name): client = Client() for c in client.containers(all=True): for container_name in c['Names']: if name == container_name: return c return None
def test_stop_test_environment_task(self): """ Test that stop task stops a running test environment """ test_id = '34fe32fdsfdsxxx' docker = Client(version='auto') image = 'adsabs/pythonsimpleserver:v1.0.0' try: container = docker.create_container( image=image, name='livetest-pythonserver-{}'.format(test_id), ) except errors.NotFound: docker.pull(image) container = docker.create_container( image='adsabs/pythonsimpleserver:v1.0.0', name='livetest-pythonserver-{}'.format(test_id), ) except Exception as error: self.fail('Unknown exception: {}'.format(error)) docker.start(container=container['Id']) stop_test_environment(test_id=test_id) self.assertFalse([i for i in docker.containers() if [j for j in i['Names'] if test_id in j]])
def main(): # register the exit signals signal.signal(signal.SIGINT, signal_handler) signal.signal(signal.SIGTERM, signal_handler) args = parse_args() global hosts_path hosts_path = args.file docker = Client(base_url='unix://%s'%args.socket) events = docker.events(decode=True) #get running containers for c in docker.containers(quiet=True,all=False): container_id = c["Id"] container = get_container_data(docker, container_id) hosts[container_id] = container update_hosts_file() #listen for events to keep the hosts file updated for e in events: status = e["status"]; if status =="start": container_id = e["id"] container = get_container_data(docker, container_id) hosts[container_id] = container update_hosts_file() if status=="stop" or status=="die" or status=="destroy": container_id = e["id"] if container_id in hosts: hosts.pop(container_id) update_hosts_file()
def main(): cli = Client(base_url="unix://var/run/docker.sock") hosts = {"static": OrderedDict(), "dynamic": OrderedDict()} for container in cli.containers(): data = container["Image"].split("/") if len(data) != 2: continue host, name = data if host == "tellendil": if name in ["static", "dynamic"]: hosts[name][container["Id"]] = container["NetworkSettings"]["Networks"]["bridge"]["IPAddress"] update_conf(safe_get(hosts, "static"), safe_get(hosts, "dynamic")) for event in cli.events(decode=True): data = event.get("from", "").split("/") if len(data) != 2: continue host, name = data if event["Action"] == "die" and host == "tellendil": if name in ["static", "dynamic"]: hosts[name].pop(event["id"]) update_conf(safe_get(hosts, "static"), safe_get(hosts, "dynamic")) elif event["Action"] == "start" and host == "tellendil": if name in ["static", "dynamic"]: hosts[name][event["id"]] = cli.inspect_container(event["id"])["NetworkSettings"]["Networks"]["bridge"]["IPAddress"] update_conf(safe_get(hosts, "static"), safe_get(hosts, "dynamic"))
def find_container(needle): cli = Client(base_url='unix://var/run/docker.sock') for container in cli.containers(): for name in container['Names']: if needle in name: return container return False
def docker_stats(): docker_client = Client(base_url='unix://var/run/docker.sock', version='1.20') data = { 'version': docker_client.version(), 'images': docker_client.images(), 'containers': [] } containers = docker_client.containers() for container in containers: _container_details = docker_client.inspect_container(container=container['Id']) _image_details = docker_client.inspect_image(image=_container_details['Image']) container['Image'] = _image_details last_log = docker_client.logs( container=container['Id'], stdout=True, stderr=True, tail=1, timestamps=True ) container['last_log'] = str(last_log, 'UTF-8').strip().split(' ')[0] data['containers'].append(container) return utils.dict_keys_to_lower(data)
class DockerOps: def __init__(self): try: config_parser = ConfigParser.ConfigParser() config_parser.read("/opt/4linux/beavops.ini") self.docker = Client(base_url=config_parser.get("docker","docker.server")) except Exception as e: logging.error("[-] Falhou ao conectar na api %s",e) def MostrarContainers(self): for d in self.docker.containers(): print d['Names'], d['Id'] def CriarContainer(self,aluno): try: logging.info("[+] Criando Container do aluno %s",aluno) container = self.docker.create_container(image="webservercloud",hostname="webservercloud", command="/bin/bash",name=aluno,tty=True) print container.get("Id") response = self.docker.start(container=container.get("Id")) except Exception as e: logging.error("[-] Falhou ao criar o container %s",e) def RemoverContainer(self,aluno): try: container = self.docker.stop(container=aluno) container = self.docker.remove_container(container=aluno) logging.info("[+] Removendo Container do aluno %s",aluno) except Exception as e: logging.error("[-] Erro ao remover o container %s",e) if __name__ == '__main__': do = DockerOps() #do.MostrarContainers() do.CriarContainer("teste")
class DockerInfo(object): def __init__(self): cfg = config.get_instance() self.cfg = cfg self.log = aLogger.getLogger(self.__class__.__name__, self.cfg) self.__url = self.cfg.get("docker.url", check_type=str) self._conn = Client(self.__url) def get_ip_info(self, name: str): name = name.split(".") try: info = self._conn.inspect_container(name[0]) if info is not None and "NetworkSettings" in info and "IPAddress" in info["NetworkSettings"]: return info["NetworkSettings"]["IPAddress"] if info["NetworkSettings"]["IPAddress"].strip() != "" else None except Exception as e: return None return None def container_name_by_ip(self, ip:str): containerIDS = self._conn.containers() for containerID in containerIDS: container_obj = self._conn.inspect_container(containerID) if container_obj is not None and "NetworkSettings" in container_obj and "IPAddress" in container_obj["NetworkSettings"]: container_ip = container_obj["NetworkSettings"]["IPAddress"] if container_obj["NetworkSettings"]["IPAddress"].strip() else None if container_ip == ip: return "%s.%s" % (container_obj["Config"]["Hostname"], container_obj["Config"]["Domainname"])
def get_containers(filter=None): """return a list of all container, or return a list that matches the filter""" docker_client = Client(base_url='unix://var/run/docker.sock') for container in docker_client.containers(all=True): container_info = docker_client.inspect_container(container['Id']) if filter is None or filter in container_info['Name']: yield container
def main(): args = parse_args( sys.argv[1:] ) kwargs = kwargs_from_env() # Connect to docker if len( kwargs.keys() ) == 0: logger.warning( 'Unable to discover Docker settings through env' ) logger.info( 'Using {}'.format( args.docker ) ) kwargs['base_url'] = args.docker # Connect to ECTD etcd_client = etcd.Client( host=args.etcd_host, port=args.etcd_port ) etcd_folder = os.path.join( args.prefix, args.domain ) logger.debug( 'Announcing to {}'.format( etcd_folder ) ) # Find the matching container docker_client = Client(**kwargs) try: containers = docker_client.containers() logger.error( containers ) except Exception as e: logger.error( e ) sys.exit( 'FAILURE - Unable to connect Docker. Is it running?' ) # Find the matching container matching = find_matching_container( containers, args ) # Main health checking loop while True: announce_services( matching.items(), etcd_folder, etcd_client, args.timeout, args.ttl, args.vulcand )
def create(self,operation): try: self.host="tcp://"+operation["host"]+":2375" version=operation["version"] client = Client(base_url=self.host) contains_num=len(client.containers(all=True)) for num in range(operation["create_num"]): name=operation["name_pro"]+str(num+contains_num) container = self.client.create_container( image=operation["image"], name=name, detach=True, stdin_open=True ) self.client.start(container=container.get('Id')) result={ "operation":"create", "return":0, "error":None } except: result={ "operation":"create", "return":-1, "error":str(traceback.format_exc()) } return result
def docker_containers(docker_url, query, all=True): """Get docker containers. :param docker_url: Docker base url :param query: container name lookup :param all: all containers or only running containers :type docker_url: str :type query: str :type all: bool :returns: all matching containers :rtype: set """ cli = Client(base_url=docker_url) results = set() containers = cli.containers(all=all) for container in containers: for name in container['Names']: name = name.split('/')[1] if name.startswith(query): results.add(name) if container['Id'].startswith(query): results.add(container['Id']) return results
def clean(self, verbose): # search for environments in systemd and apache config as well as in all docker containers envsSystemd = [] for root, dirs, files in os.walk('/etc/systemd/system'): envsSystemd.extend([x[7:-8] for x in files if x.startswith('docker-%s-' % self.name)]) envsApache = [] for root, dirs, files in os.walk('/etc/httpd/conf.d/sites-enabled'): envsApache.extend([x[:-5] for x in files if x.startswith('%s-' % self.name)]) cli = Client(base_url = 'unix://var/run/docker.sock') envsDocker = [] for c in cli.containers(all = True): e = [x[1:] for x in c['Names'] if x.startswith('/%s-' % self.name) and Param.isValidName(x[(2 + len(self.name)):])] if len(e) == 1: envsDocker.extend(e) envs = [] envs.extend(envsSystemd) envs.extend(envsApache) envs.extend(envsDocker) toBeRemoved = set(envs) - set([e.Name for e in self.environments]) if len(toBeRemoved) > 0 and verbose: print ' %s' % self.name for i in toBeRemoved: call = subprocess.Popen(['sudo', '-u', 'root', 'docker-clean', i], stdout = subprocess.PIPE, stderr=subprocess.PIPE) (out, err) = call.communicate() if verbose or err != '': print ' %s' % i print '\n'.join([' %s' % x for x in out.split('\n')]) print '\n'.join([' %s' % x for x in err.split('\n')])
def updateBuild(strURL): processBuild(strURL) cli = Client('unix://var/run/docker.sock') containers = cli.containers() images = cli.images() pContID = SERVER_CONFIG.get('PRIMARY_CONTAINER','id') pCmnd = SERVER_CONFIG.get('PRIMARY_CONTAINER','command') sContID = SERVER_CONFIG.get('SECONDARY_CONTAINER','id') for container in containers: if container["Id"] == pContID: createdCont = cli.create_container(image=container["Image"], command=str(pCmnd), tty=True) response = cli.start(container=createdCont.get('Id')) sContID = str(createdCont.get('Id')) print " Started copy of primary container " + str(createdCont.get('Id')) + " Now going to redirect traffic! " continue secIP = cli.inspect_container(sContID) lJsecIp = secIP['NetworkSettings']['IPAddress'] #dJsecIp = secIP.pop(0) print str(lJsecIp) #redirecSecTraffic() #scriptExecutor(pContID) #redirecPriTraffic() print " Stopping and removing secondary container created " cli.stop(sContID) remRsp = cli.remove_container(sContID, force=True) print str(remRsp) return "Hello World" + str("\n") + str(containers) + str(pContID) + str(sContID) + str(response)
def run(self): if self.engine == "K3": return # For K3: poll docker to find the mesos container. while not self.containersKnown() and not self.finished: for m in self.machines: if self.containers[m] == None: c = Client(base_url='tcp://' + m + ':41000', version='1.15') allContainers = c.containers() allNames = [c['Names'][0] for c in allContainers] mesos = [n for n in allNames if 'mesos' in n] if len(mesos) > 0: self.containers[m] = mesos[0] #print("Container named %s found on %s" % (mesos[0], m)) time.sleep(1) #print("\tCollecting '" + self.engine + "' container on '" + str(self.machines) + "'") con = psycopg2.connect(host="qp1", database="postgres", user="******", password="******") cur = con.cursor() while not self.finished: myobjs = self.get_data() if myobjs: for (myobj, machine) in zip(myobjs, self.machines): self.parse_data(cur, con, self.trial_id, myobj.values()[0]['stats'][0], machine) time.sleep(1)
def main(): """Remove containers which elapsed 1hour since it created""" client = Client("tcp://docker:2375") for container in client.containers(all=True): if container["Image"] == "wh11e7rue/tick.tock.tech-session" and "hour" in container["Status"]: client.remove_container(container["Id"], force=True) print("Removed:", container["Id"], container["Status"])
def all_container(): docker_client = Client(base_url='tcp://%s:2375' % local_ip,version='1.20', timeout=10) docker_container=docker_client.containers(all=True) container=[] for i in docker_container: container.append({'Names':i['Names'][0].encode('utf-8').replace('/',''),'Status':i['Status'].encode('utf-8')}) return container
def main(): pod_name = os.environ.get("POD_NAME", None) if not pod_name: print 'make sure POD_NAME is injected to the container as an env variable' sys.exit(1) cli = Client(base_url='unix://var/run/docker.sock') containers = cli.containers(filters={'label': 'io.kubernetes.pod.name='+pod_name}) container_id = None for container in containers: if container['Labels']['io.kubernetes.container.name'] == 'nginx': container_id = container['Id'] break # if container['Labels']['io.kubernetes.container.name'] not in ('cdsmon-collectd', 'POD'): # container_id = container['Id'] # break if not container_id: print 'cannot pick container id automatically' sys.exit(1) cmd = cli.exec_create(container=container_id, cmd='hostname') print cli.exec_start(cmd)
def main(): while True: c = Client(base_url='unix://var/run/docker.sock') containers=c.containers() config=get_config() print config locate=config['locate'] host=config['node'] port=config['port'] client = etcd.Client( host=host, port=port, allow_reconnect=True,) for k in containers: a= get_container_key(k) if a['virtual_host']!= None: path=locate+'/'+a['virtual_host'] locate_write=path+'/'+a['container_name'] value_write={"address":a['Host_IP'],"port":a['container_port']} value_write_json=json.dumps(value_write) try: IfPathExist=client.read(path) except etcd.EtcdKeyNotFound: IfPathExist=False if IfPathExist==False: client.write(path,0,dir=True,ttl=64) print time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time()))+" creat dir "+path client.write(locate_write,value_write_json,ttl=64) else: client.write(locate_write,value_write_json,ttl=64) print time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time()))+" put key "+locate_write+" value "+value_write_json time.sleep(5)
def listContainers(self): containers = [] cli = Client(self.__docker_url,version='auto') for container in cli.containers(all=True): container = Container(container.get("Names"), container.get("Status")) containers.append(container) return containers
def main(): global CONTAINER_NAME c = Client(base_url='unix://var/run/docker.sock') if len(sys.argv) < 2: name = CONTAINER_NAME else: name = sys.argv[1] #for remove test container's image if name == 'remove': print 'removing %s images' % IMAGE_NAME if IMAGE_NAME not in [y.encode('utf-8').replace('/', '') for x in c.containers(all=True) for y in x['Names']]: print 'image is not exist' return c.remove_image(image=IMAGE_NAME, force=True, noprune=False) print '%s images removed' % CONTAINER_NAME return #print 'check container name: %s' % name #if not recreate_volume_container(c, name): # if not check_image_name(c, IMAGE_NAME): # build_image(c, IMAGE_NAME) # create_container(c, name) if not check_image_name(c, IMAGE_NAME): build_image(c, IMAGE_NAME) recreate_volume_container(c, CONTAINER_NAME)
def collectData(self): cli = Client(base_url='unix://var/run/docker.sock') containers_obj = cli.containers() #containers_obj = cli.containers(all='true') self.mon_dt = [] for container in containers_obj: cont_info = {} cont_info['name'] = container['Names'] cont_info['image_name'] = container['Image'] cont_info['created'] = container['Created'] cont_info['id'] = container['Id'] cont_info['image'] = container['ImageID'] #cont_info['stats'] = cli.stats(cont_info['id'], False, False) cont_info['status'] = self.statusCode(container['Status']) if cont_info['status'] == 1: cont_info['stats'] = self.getstats(cont_info['id']) else: con = {} con['id'] = 0 con['cpu_perc'] = 0 con['mem_perc'] = 0 con['mem_usage_MB'] = 0 con['mem_limit_MB'] = 0 con['net_rx_MB'] = 0 con['net_tx_MB'] = 0 con['block_in_MB'] = 0 con['block_ou_MB'] = 0 cont_info['stats'] = con self.mon_dt.append(cont_info)
def test_docker(): cli = Client(base_url='unix://var/run/docker.sock') response = cli.containers() assert isinstance(response, list), 'Ensure you have sufficiant' + \ 'credentials to use docker with' + \ 'your current user'
with open(inspect_file, 'w') as fp: fp.write(json.dumps(inspect, indent=2)) try: # We need to keep a referrence to the process so we can remove the reference # when they are done. See comment below. # We also need to spawn the process in the backround, therefor, Popen, not call. procs.append( subprocess.Popen([runner_file, value, docker_id, status])) # Delete the references to completed processes so python will do garbage collection # to kill old processes. They will be zombies from the time they are done to when # this code is run. A zombie doesnt use any resources, but its nice to get rid of them. procs[:] = [proc for proc in procs if proc.poll() is None] except OSError: # Probably not executable.. return for i in c.containers(filters={'status': 'running'}): handler(docker_id=i['Id'], status='running') # c.events() returns a blocking generator for event in c.events(decode=True): try: handler(docker_id=event['id'], status=event['status']) except Exception, e: # We shoulnt die, but log.. sleep 1 second so we wont loop.. # Many things can go wrong in the handler logging.exception(e) time.sleep(1)
def remove_image(): client = DockerClient(version="1.15") for container in client.containers(): if container[u"Image"] == tag + ":latest": client.remove_container(container[u"Names"][0]) client.remove_image(tag, force=True)
host = "127.0.0.1" port = 1514 while True: try: s.connect((host,port)) if s.getpeername(): print "Connected to {0}".format(s.getpeername()) break except: print "Port 1514 is not ready, sleep for a moment" sleep(3) c = Client(base_url='unix://var/run/docker.sock') while True: run_c_list = c.containers() for container in run_c_list: container_found = False for existing_container in c_list: if container["Id"] == existing_container["Id"]: container_found = True break if container_found == False: if "Labels" in container and "io.kubernetes.pod.namespace" in container["Labels"]: if container["Labels"]["io.kubernetes.pod.namespace"] != "kube-system": #print json.dumps(container,indent=2) c_list.append(container) print "Monitor log for new container : {0}".format(container["Id"]) thread = Thread(target = print_logs, args = (container,s,))
class ContainerHandler(ResourceHandler): @classmethod def is_available(self, io): return True def __init__(self, agent, io=None): super().__init__(agent, io) self._client = None def pre(self, ctx, resource: Container): self._client = Client(base_url="unix://var/run/docker.sock") def post(self, ctx, resource: Container): self._client.close() def check_resource(self, ctx, resource: Container): current = resource.clone() containers = self._client.containers(all=True) docker_resource = None for container in containers: names = container["Names"] search_name = resource.name if search_name[0] != "/": search_name = "/" + search_name if search_name in names: docker_resource = container if docker_resource is None: current.state = "purged" return current else: data = self._client.inspect_container(docker_resource["Id"]) current.state = data["State"]["Status"] ctx.set("container_id", docker_resource["Id"]) return current def do_changes(self, ctx, resource: Container, changes) -> bool: """ Enforce the changes """ if "state" in changes: state = changes["state"] if state["current"] == "purged" and state["desired"] == "running": # ensure the image is pulled images = self._client.images(name=resource.image) if len(images) == 0: msg = self._client.pull(resource.image) if "not found" in msg: raise Exception("Failed to pull image %s: %s" % (resource.image, msg)) cont = self._client.create_container( image=resource.image, command=resource.command, detach=resource.detach, host_config={"memory_limit": resource.memory_limit}) self._client.start(cont["Id"]) self._client.rename(cont["Id"], resource.name) ctx.set_created() elif state["desired"] == "purged": container_id = ctx.get("container_id") if state["current"] == "running": self._client.stop(container_id) self._client.remove_container(container_id) ctx.set_purged() def facts(self, resource: Container): """ Get facts about this resource """ return {}
from docker import Client from docker.tls import TLSConfig tls_config = TLSConfig(client_cert=('/root/.sdc/docker/dissipate/cert.pem', '/root/.sdc/docker/dissipate/key.pem'), verify='/root/.sdc/docker/dissipate/ca.pem') cli = Client(base_url='https://us-sw-1.docker.joyent.com:2376', tls=tls_config) print str(cli.containers())
class MissionTask(BaseTask): docker_id = None def __init__(self): self.cli = Client(base_url='unix://var/run/docker.sock') def _host_config(self, instance_id, tech): ssh_port = 10000 + instance_id * 2 - 1 tech_port = 10000 + instance_id * 2 if tech == 'tomcat': host_config = self.cli.create_host_config(port_bindings={ 22: ssh_port, 8080: tech_port, }) ports = [22, 8080] ports_detail = "{0}->{1}|{2}->{3}".format(22, ssh_port, 8080, tech_port) elif tech == 'nodejs': host_config = self.cli.create_host_config(port_bindings={ 22: ssh_port, 3000: tech_port, }) ports = [22, 3000] ports_detail = "{0}->{1}|{2}->{3}".format(22, ssh_port, 3000, tech_port) elif tech == 'php': host_config = self.cli.create_host_config(port_bindings={ 22: ssh_port, 80: tech_port, }) ports = [22, 80] ports_detail = "{0}->{1}|{2}->{3}".format(22, ssh_port, 80, tech_port) else: host_config = self.cli.create_host_config(port_bindings={ 22: ssh_port, }) ports = [22] ports_detail = "{0}->{1}".format(22, ssh_port) return ports, host_config, ports_detail def _start_docker(self, docker_id): try: self.cli.start(docker_id) return True except: return False def _stop_docker(self, docker_id): try: self.cli.stop(docker_id) return True except: return False def _rm_docker(self, docker_id): try: self.cli.remove_container(docker_id, force=True) return True except: return False def _create_docker(self, instance_id, tech): ports, host_config, ports_detail = self._host_config(instance_id, tech) result = self.cli.create_container('lemonbar/centos6-ssh', ports=ports, dns='', working_dir='', host_config=host_config, name='docker-shqs-{0}'.format( instance_id, )) docker_id = result.get('Id') logger.debug(result) if self._start_docker(docker_id): aLL_docker = self.cli.containers(all=True) docker_new = [x for x in aLL_docker if x.get('Id') == docker_id][0] status = docker_new.get('status') else: status = 'unknown' Docker_list.objects.filter(id=instance_id).update(docker_id=docker_id, status=status, ports=ports_detail) return result def run(self, instance_id, command): docker_instance = Docker_list.objects.get(id=instance_id) tech = docker_instance.tech.content if command == 'create': result = self._host_config(instance_id, tech) elif command == 'start': result = self._start(docker_instance.docker_id) elif command == 'stop': result = self._stop(docker_instance.docker_id) elif command == 'rm': result = self._rm_docker(docker_instance.docker_id) else: result = 'not exist command' logger.debug(result)
def file_queue(path, base_dir="/var/lib/docker/data/"): """ Processes files that have been added from the rq-worker, and tells vent-management to start plugins that match the mime type for the new file. """ import ConfigParser import magic import os import time from docker import Client from docker.utils.types import LogConfig c = Client(base_url='unix://var/run/docker.sock') # first some cleanup containers = c.containers(quiet=True, all=True, filters={'status': "exited"}) for cont in containers: try: # will only remove containers that aren't running c.remove_container(cont['Id']) except Exception as e: pass template_dir = base_dir + "templates/" plugins_dir = base_dir + "plugins/" vent_dir = "/vent/" if base_dir != "/var/lib/docker/data/": vent_dir = base_dir try: config = ConfigParser.RawConfigParser() # needed to preserve case sensitive options config.optionxform = str # Check file exists if os.path.isfile(template_dir + 'modes.template'): config.read(template_dir + 'modes.template') plugin_array = [] # Check section exists and has options if config.has_section("plugins") and config.options("plugins"): plugin_array = config.options("plugins") plugins = {} for plug in plugin_array: if plug not in ["core", "visualization", "collectors"]: plugins[plug] = config.get("plugins", plug) container_count = 0 container_max = 50 try: config = ConfigParser.RawConfigParser() # needed to preserve case sensitive options config.optionxform = str if os.path.isfile(template_dir + 'core.template'): config.read(template_dir + 'core.template') if config.has_section("active-containers") and config.has_option( "active-containers", "count"): container_max = int(config.get("active-containers", "count")) except Exception as e: print(str(e)) file_mime = None try: f_path = path.split("_", 1)[1] file_mime = magic.from_file(f_path, mime=True) except Exception as e: print(str(e)) for plugin in plugins: # check mime types mime_types = [] try: config = ConfigParser.RawConfigParser() # needed to preserve case sensitive options config.optionxform = str if os.path.isfile(template_dir + plugin + '.template'): config.read(template_dir + plugin + '.template') if config.has_section("service") and config.has_option( "service", "mime_types"): mime_types = config.get("service", "mime_types").split(",") except Exception as e: print(str(e)) # check file extensions ext_types = [] try: config = ConfigParser.RawConfigParser() # needed to preserve case sensitive options config.optionxform = str if os.path.isfile(template_dir + plugin + '.template'): config.read(template_dir + plugin + '.template') if config.has_section("service") and config.has_option( "service", "ext_types"): ext_types = config.get("service", "ext_types").split(",") except Exception as e: print(str(e)) if len(mime_types) == 0 or file_mime in mime_types or path.split( ".", -1)[-1] in ext_types: # check resources before creating container # wait until there are resources available container_count = len( c.containers(filters={'status': 'running'})) while container_count >= container_max: time.sleep(5) container_count = len( c.containers(filters={'status': 'running'})) cmd = "python2.7 " + vent_dir + "template_parser.py " + plugin + " start " + path os.system(cmd) except Exception as e: print(str(e)) return
splunk_docker_utils.do_setup() # We need to loop running containers, getting stats hosts = open (splunk_docker_utils.RUN_DIR + '/docker_hosts.dat','r') for host in hosts: host=host.rstrip() conn_string="tcp://" + host +":2375" #cli = Client(base_url='tcp://192.168.160.166:2375', version='auto') try: cli = Client(base_url=conn_string, version='auto') containers=cli.containers() count=0 for container in containers: #print "container is "+str(container) name_attr=re.search( r'\[u\'\/(\w+)\'\]',str(containers[count]['Names'])) container_name=name_attr.group(1) count +=1 inspect=cli.inspect_container(container_name) # We need hostname, unfortunately we have to inspect container for it hostname=inspect['Config']['Hostname'] stats=cli.stats(container_name,decode=None, stream=False) # Network #networks": { # "eth0": {
class Cleaner: def __init__(self, configs): self.configs = configs self.client = Client(base_url=self.configs['docker_endpoint']) """ Check the connection with docker. End execution if there's no connection """ try: self.client.ping() except: Logger.log( "Could not connect to Docker. Verify if docker is running or your settings file" ) exit() if self.configs['clear_containers']: self.clear_containers() if self.configs['clear_images']: self.clear_images() def should_remove_container(self, status): """ Given that we want to remove all obsolet containers (=not running) return True to all container statuses not containing 'Up' """ return not status.startswith('Up') def remove_container(self, container): """ Try to remove container given the container Id """ try: self.client.remove_container(container=container['Id']) Logger.log("Removing container: %s" % container['Id']) except: Logger.log_container_error("container", container) def should_remove_image(self, image, containers): """ Determine if the image should be removed if there is no containers based on that image running at the momment """ for container in containers: if container['Image'] in image['RepoTags']: return False return True def remove_image(self, image): """ Try to remove image given the image Id """ try: self.client.remove_image(image=image['Id'], force=True) Logger.log("Removing image: %s" % image['Id']) except: Logger.log_container_error("image", image) def clear_containers(self): """ Search for all containers in order to remove those not running """ for container in self.client.containers(all=True): if self.should_remove_container(container['Status']): self.remove_container(container) def clear_images(self): """ Search for images in order to remove those which doesn't have containers instances running """ running_containers = self.client.containers() for image in self.client.images(): if self.should_remove_image(image, running_containers): self.remove_image(image)
class DockerSpy: """ # Usage: docker = dockerspy(container_api_url='unix://var/run/docker.sock') # print docker information print(docker.describe_containers) print(docker.env(container='chargeback-sync')) print(docker.virtual_port(container='chargeback-sync')) print(docker.ip_address(container='chargeback-sync')) # listen to events from docker API, similar to docker events docker.events() # print all containers information as a dictionary structure docker.containers """ def __init__(self, container=None, container_api_url='unix://var/run/docker.sock'): """ :rtype: object :type dockerspy: object """ self._client = Client(base_url=container_api_url) self.containers = {} if container: self.containers[container] = self._client.inspect_container( container=container) self.tagged_containers = {} def get_containers(self): """ :rtype : list :return: containers """ try: # list live containers names using list comprehension containers = [ container['Names'][0].replace('/', '') for container in self._client.containers() ] return containers except Exception: raise @staticmethod def _into_dict(kv_config): env_dict = {} for item in kv_config: k, v = item.split('=', 1) env_dict[k] = v return env_dict @property def describe_containers(self): # update self.containers for name in self.get_containers(): self.containers[name] = self._client.inspect_container(name) self.containers[name]['Config']['Env'] = self._into_dict( kv_config=self.containers[name]['Config']['Env']) return self.containers def events(self): """ """ try: return self._client.events(decode=True) except KeyboardInterrupt: pass def env(self, container=None): """ :rtype: dict :return: container's environment variable """ """ # internal method to transform k=v environment variable into dict def _into_dict(kv_config): env_dict = {} for item in kv_config: k, v = item.split('=', 1) env_dict[k] = v return env_dict """ # If self.cotnainers exists or grater then 0 return Config.Env if self.containers: return self.containers[container]['Config']['Env'] else: return self.describe_containers[container]['Config']['Env'] def virtual_port(self, container): # internal method to transform k=v environment variable into dict """ def _into_dict(kv_config): env_dict = {} for item in kv_config: k, v = item.split('=', 1) env_dict[k] = v return env_dict """ # return config env of a container in dictionary format if self.containers: return self.env(container)['VIRTUAL_PORT'] else: return self.describe_containers[container]['Config']['Env'][ 'VIRTUAL_PORT'] def ip_address(self, container): if self.containers: return self.containers[container]['NetworkSettings']['IPAddress'] else: return self.describe_containers[container]['NetworkSettings'][ 'IPAddress'] def memberof(self, service_name): pass
client = Client(base_url='unix://var/run/docker.sock') #Stop and remove #client.stop("python-container") #client.remove_container("python-container") # create and start the container client.create_container(name="python-container", image="ubuntu:latest", detach=True, tty=True) client.start("python-container") # Get the PID container = client.containers(filters={"name": "python-container"}) inspect = client.inspect_container("python-container") PID = inspect['State']['Pid'] print("Printing PID: ") print PID print("Move the namespace to /var/run/netns/") proc_pid = "/proc/" + str(PID) + "/ns/net" var_run_pid = "/var/run/netns/" + str(PID) print proc_pid print var_run_pid subprocess.Popen(["sudo", "ln", "-s", proc_pid, var_run_pid], stdout=subprocess.PIPE) client.stop("python-container")
class DockerRunner(object): """ Responsible for pulling a docker image, creating a container, running the container, then tearing down the container """ service_name = None def __init__(self, image, name, command=None, environment=None, **kwargs): """ :param image: full name of the docker image to pull :param name: name of the container in `docker run --name` :param command: command for the container in `docker run <> command` :param enviroment: environment variables in the container :param mem_limit: Memory limit to enforce on the container :param kwargs: keyword args to pass direclty to docker.utils.create_host_config """ self.image = image self.name = name self.command = command self.environment = environment self.running_properties = None self.time_out = 30 self.pull = kwargs.get('pull', True) self.running_port = None self.running_host = None self.client = Client(version='auto') kwargs.pop('build_requirements', '') self.host_config = self.client.create_host_config(**kwargs) self.container = None try: self.logger = current_app.logger except RuntimeError: # Outside of application context self.logger = logging.getLogger("{}-builder".format(self.name)) self.logger.setLevel(logging.DEBUG) self.setup() def setup(self): """ Pull the image and create a container with host_config """ exists = [i for i in self.client.images() if self.image in i['RepoTags']] # Only pull the image if we don't have it if not exists or self.pull: self.client.pull(self.image) self.logger.debug("Pulled {}".format(self.image)) self.container = self.client.create_container( image=self.image, host_config=self.host_config, name=self.name, command=self.command, environment=self.environment ) self.logger.debug("Created container {}".format(self.container['Id'])) def start(self, callback=None): """ Starts the container, and optionally executes a callback that is passed the container's info :param callback: Optional callback function that is called after the container is up. Useful for running more fine-grained container provisioning. :return: None """ self.logger.debug('Starting container {}'.format(self.image)) response = self.client.start(container=self.container['Id']) if response: self.logger.warning(response) self.logger.debug('Checking if {} service is ready'.format(self.name)) timed(lambda: self.running, time_out=30, exit_on=True) timed(lambda: self.ready, time_out=30, exit_on=True) self.logger.debug('Service {} is ready'.format(self.name)) if callable(callback): callback(container=self.container) self.logger.debug('Startup of {} complete'.format(self.name)) def teardown(self): """ Stops and removes the docker container. :return: None """ response = self.client.stop(container=self.container['Id']) if response: self.logger.warning(response) response = self.client.remove_container(container=self.container['Id']) if response: self.logger.warning(response) try: timed(lambda: self.running, time_out=self.time_out, exit_on=False) except TimeOutError: self.logger.warning( 'Container teardown timed out, may still be running {}' .format(self.container) ) print 'Timeout' @property def ready(self): """ Check the service is ready """ return True @property def running(self): """ Determine if the container started :return: boolean """ running_properties = [i for i in self.client.containers() if i['Id'] == self.container['Id']] if len(running_properties) == 0 or 'Up' not in running_properties[0].get('Status', ''): return False else: self.logger.info('Docker container {} running'.format(self.image)) self.running_properties = running_properties if self.service_name: try: running = self.client.port(self.container, config.DEPENDENCIES[self.service_name.upper()]['PORT']) self.running_host = running[0]['HostIp'] self.running_port = running[0]['HostPort'] except KeyError: pass return True def provision(self, services, requirements=None): """ Run the provisioner of this class for a set of services :param services: list of services """ if hasattr(self, 'service_provisioner'): provisioner = self.service_provisioner(services=services, container=self, requirements=requirements) provisioner()
class DockerApiDriver(API): def __init__(self): """ Initialize client """ self.name = None self.home = None self.handle = None super(API, self).__init__() def connect(self, **connectargs): """ Create Client handle to connnect to Docker server """ try: for key in connectargs: vars(self)[key] = connectargs[key] self.name = self.options['name'] for key in self.options: if key == "home": self.home = self.options['home'] break if self.home is None or self.home == "": self.home = "/var/tmp" self.handle = super(DockerApiDriver, self).connect() self.dockerClient = Client(base_url='unix://var/run/docker.sock') return self.handle except Exception as e: main.log.exception(e) def getListOfImages(self, repo="onosproject/onos"): """ Get the list of image tags """ try: imageList = list(self.dockerClient.images(name=repo)) imageListToSend = [] duplicateTagDetected = 0 for imageDict in imageList: if imageDict['RepoTags'] is not None: if len(imageDict['RepoTags']) > 1: duplicateTagDetected = 1 imageListToSend.append( imageDict['RepoTags'][0].encode('UTF8').split(':')[1]) return imageListToSend, duplicateTagDetected except Exception as e: main.log.exception(e) def dockerPull(self, onosRepo="onosproject/onos", onosTag="latest"): """ Pulls Docker image from repository """ try: main.log.info(self.name + ": Pulling Docker image " + onosRepo + ":" + onosTag) for line in self.dockerClient.pull( repository = onosRepo, \ tag = onosTag, stream = True ): print "#", main.log.info(json.dumps(json.loads(line), indent=4)) #response = json.dumps( json.load( pullResult ), indent=4 ) if re.search("for onosproject/onos:" + onosTag, line): main.log.info("onos docker image pulled is: " + line) return main.TRUE else: main.log.error("Failed to download image from: " + onosRepo + ":" + onosTag) main.log.error("Error respone: ") main.log.error(line) return main.FALSE except Exception: main.log.exception(self.name + ": Uncaught exception!") main.cleanup() main.exit() def dockerCreateCT(self, onosImage="onosproject/onos:latest", onosNode="onos1"): """ Create a Docker container with a specific image """ try: main.log.info(self.name + ": Creating Docker container for node: " + onosNode) response = self.dockerClient.create_container( image=onosImage, \ tty=True, name=onosNode, detach=True ) #print response #print response.get("Id") #print response.get("Warnings") if (str(response.get("Warnings")) == 'None'): main.log.info("Created container for node: " + onosNode + "; container id is: " + response.get("Id")) return (main.TRUE, response.get("Id")) else: main.log.info("Noticed warnings during create") return (main.FALSE, null) except Exception: main.log.exception(self.name + ": Uncaught exception!") main.cleanup() main.exit() def dockerStartCT(self, ctID): """ Start Docker container """ try: main.log.info(self.name + ": Starting Docker conatiner Id " + ctID) response = self.dockerClient.start(container=ctID) if response is None: main.log.info("Started container for Id: " + ctID) return main.TRUE else: main.log.info("Noticed warnings during start") return main.FALSE except Exception: main.log.exception(self.name + ": Uncaught exception!") main.cleanup() main.exit() def dockerStopCT(self, ctName): """ Stop docker container """ try: main.log.info(self.name + ": Stopping Docker conatiner for node " + ctName) response = self.dockerClient.stop(ctName) if response is None: main.log.info("Stopped container for node: " + ctName) return main.TRUE else: main.log.info("Noticed warnings during stop") return main.FALSE except errors.NotFound: main.log.info(ctName + " not found! Continue on tests...") return main.TRUE except Exception: main.log.exception(self.name + ": Uncaught exception!") #main.cleanup() #main.exit() def dockerRestartCT(self, ctName): """ Restart Docker container """ try: main.log.info(self.name + ": Restarting Docker conatiner for node " + ctName) response = self.dockerClient.restart(ctName) if response is None: main.log.info("Restarted container for node: " + ctName) return main.TRUE else: main.log.info("Noticed warnings during Restart") return main.FALSE except Exception: main.log.exception(self.name + ": Uncaught exception!") main.cleanup() main.exit() def dockerCheckCTName(self, ctName): """ Check Docker conatiner status """ try: main.log.info(self.name + ": Checking Docker Status for CT with 'Names' " + ctName) namelist = [ response["Names"] for response in self.dockerClient.containers(all=True) if not [] ] main.log.info("Name list is: " + str(namelist)) if ([ctName] in namelist): main.log.info("Container " + ctName + " exists") return main.TRUE else: main.log.info("Container " + ctName + " does not exist") return main.FALSE except errors.NotFound: main.log.warn(ctName + "not found! Continue with the tests...") return main.FALSE except Exception: main.log.exception(self.name + ": Uncaught exception! Continue tests...") #main.cleanup() #main.exit() def dockerRemoveCT(self, ctName): """ Remove Docker conatiner """ try: main.log.info(self.name + ": Removing Docker container for node " + ctName) response = self.dockerClient.remove_container(ctName, force=True) if response is None: main.log.info("Removed container for node: " + ctName) return main.TRUE else: main.log.info("Noticed warnings during Remove " + ctName) return main.FALSE main.log.exception(self.name + ": not found, continuing...") except errors.NotFound: main.log.warn(ctName + "not found! Continue with the tests...") return main.TRUE except Exception: main.log.exception(self.name + ": Uncaught exception! Continuing...") #main.cleanup() #main.exit() def dockerRemoveImage(self, imageRepoTag=None): """ Remove Docker image """ rmResult = main.TRUE if self.dockerClient.images() is []: main.log.info("No docker image found") return rmResult else: imageList = [ image["Id"] for image in self.dockerClient.images() if image["RepoTags"] is None or imageRepoTag in image["RepoTags"] ] for id in imageList: try: main.log.info(self.name + ": Removing Docker image " + id) response = self.dockerClient.remove_image(id, force=True) if response is None: main.log.info("Removed Docker image: " + id) rmResult = rmResult and main.TRUE else: main.log.info("Noticed warnings during Remove " + id) rmResult = rmResult and main.FALSE except errors.NotFound: main.log.warn(image + "not found! Continue with the tests...") rmResult = rmResult and main.TRUE except Exception: main.log.exception(self.name + ": Uncaught exception! Continuing...") rmResult = rmResult and main.FALSE #main.cleanup() #main.exit() return rmResult def fetchLatestClusterFile(self, branch="master"): """ Fetch onos-form-cluster file from a particular branch """ try: command = "wget -N https://raw.githubusercontent.com/opennetworkinglab/\ onos/" + branch + "/tools/package/bin/onos-form-cluster" subprocess.call(command) # output checks are missing for now command = "chmod u+x " + "onos-form-cluster" subprocess.call(command) return main.TRUE except Exception: main.log.exception(self.name + ": Uncaught exception!") main.cleanup() main.exit() def onosFormCluster(self, onosIPs, cmdPath, user="******", passwd="karaf"): """ From ONOS cluster for IP addresses in onosIPs list """ try: onosIPs = " ".join(onosIPs) command = "{}/onos-form-cluster -u {} -p {} {}".format( cmdPath, user, passwd, onosIPs) result = subprocess.call(command, shell=True) if result == 0: return main.TRUE else: main.log.info("Something is not right in forming cluster>") return main.FALSE except Exception: main.log.exception(self.name + ": Uncaught exception!") main.cleanup() main.exit() def dockerIP(self, ctName): """ Fetch IP address assigned to specified node/container """ try: output = self.dockerClient.inspect_container(ctName) nodeIP = output['NetworkSettings']['IPAddress'] main.log.info(" Docker IP " + str(nodeIP)) return str(nodeIP) except Exception: main.log.exception(self.name + ": Uncaught exception!") main.cleanup() main.exit()
import json from docker import Client cli = Client(base_url='unix://var/run/docker.sock') list_containers = cli.containers(all=True) json_name = "Names" json_id = "Id" cpu="cpu_stats" memory="memory_stats" precpu="precpu_stats" for container in list_containers: print "Container ID and info:" container_id = container[json_id] print container_id # print json.dumps(container, sort_keys=True, indent=4, separators=(',',': ')) container_name = container[json_name][0] print container_name print print "Container stats:" stats = cli.stats(container, stream=False) # print json.dumps(stats, sort_keys=True, indent=4, separators=(',',': ')) print "CPU_STATS:" print json.dumps(stats[cpu], sort_keys=True, indent=4, separators=(',',': ')) print "MEMORY_STATS:" print json.dumps(stats[memory], sort_keys=True, indent=4, separators=(',',': ')) print "PRE_CPU_STATS:" print json.dumps(stats[precpu], sort_keys=True, indent=4, separators=(',',': ')) print
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', filename=(args.log_file if args.log_file != '-' else None)) if args.zone is None: args.zone = args.domain logging.info("Starting with arguments %s", args) c = Client(**(kwargs_from_env())) resolver = Resolver() resolver.nameservers = [args.server] if args.catchup: logging.info("Registering existing containers") containers = c.containers() for container in containers: register_container(container["Id"]) # TODO use docker-py streaming API events_pipe = Popen(['docker', 'events'], stdout=PIPE) while True: line = events_pipe.stdout.readline() if line != '': text_line = line.decode().rstrip() logging.debug("Read line %s", text_line) m = re.search(r".*\s+container\s+([a-z]+)\s+([0-9a-f]{64}).*$", text_line) if m: event = m.group(1)
import os import socket from docker import Client if __name__ == "__main__": # try to resolve valid hostname for the web service entry_counter = 2 if os.environ.get("INSTANCE_NAME", None): container_prefix = os.environ.get("INSTANCE_NAME", "production") else: # try to get the container name from the docker file cli = Client(base_url='unix://var/run/docker.sock') container_prefix = None for c in cli.containers(): if c["Id"].startswith(socket.gethostname()): container_prefix = c["Names"][-1].split("_", 1)[0].lstrip("/") break hostname_prefix = container_prefix + "_" if container_prefix else "" hosts = ["%sweb_1" % hostname_prefix] while True: try: hostname = "%sweb_%d" % (hostname_prefix, entry_counter) socket.gethostbyname_ex(hostname) hosts.append(hostname) entry_counter += 1 except:
class Collector: def __init__(self): self.logger = logging.getLogger("Monitor") self.INFLUXDB = {} self.INFLUXDB['HOST'] = os.getenv("INFLUXDB_HOST", "localhost") self.INFLUXDB['PORT'] = os.getenv("INFLUXDB_PORT", "8086") self.INFLUXDB['USERNAME'] = os.getenv("INFLUXDB_USERNAME", "root") self.INFLUXDB['PASSWORD'] = os.getenv("INFLUXDB_PASSWORD", "root") self.INFLUXDB['DBNAME'] = os.getenv("INFLUXDB_DBNAME", "autoscaling") self.INFLUXDB['SERIES'] = os.getenv("INFLUXDB_SERIES", "monitoring") self.COLLECT_TIME_INTERVAL = int( os.getenv("COLLECT_TIME_INTERVAL", "3")) self.BATH_TIME_INTERVAL = int(os.getenv("BATH_TIME_INTERVAL", "10")) self.influxdb_client = InfluxDBClient( host=self.INFLUXDB['HOST'], port=self.INFLUXDB['PORT'], username=self.INFLUXDB['USERNAME'], password=self.INFLUXDB['PASSWORD'], database=self.INFLUXDB['DBNAME']) self.docker_client = Client(base_url='unix://var/run/docker.sock') self.data_bath = [] self.current_data = 0 def collecting(self, container_id): mesos_task_id = "" app_name = "" container_envs = self.docker_client.inspect_container( container_id)['Config']['Env'] for env in container_envs: if env.startswith('MESOS_TASK_ID'): mesos_task_id = env.split('=')[1] index = mesos_task_id.rfind('.') app_name = mesos_task_id[:index] break while True: try: stat = self.docker_client.stats(container_id, decode="utf8", stream=False) cpu_usage = ( stat["cpu_stats"]["cpu_usage"]["total_usage"] - stat["precpu_stats"]["cpu_usage"]["total_usage"] ) * 100 / math.pow(10, 9) / len( stat["cpu_stats"]["cpu_usage"]["percpu_usage"]) mem_usage = stat["memory_stats"]["usage"] / stat[ "memory_stats"]["limit"] * 100 current_time = datetime.now().timestamp() data = [ current_time, container_id, app_name, mesos_task_id, cpu_usage, mem_usage ] self.logger.debug("Append: " + str(data)) self.data_bath.append(data) time.sleep(self.COLLECT_TIME_INTERVAL) except errors.NotFound as e: self.logger.info( "Container {} has gone away".format(container_id)) break except Exception as e: self.logger.error("Error " + str(e)) break def send_data(self): while True: try: time.sleep(self.BATH_TIME_INTERVAL) if self.data_bath: data = dict() data['name'] = self.INFLUXDB["SERIES"] data['columns'] = [ 'time', 'container_id', 'app_uuid', 'mesos_task_id', "cpu_usage", "mem_usage" ] data['points'] = self.data_bath self.data_bath = [] self.logger.info("Send data ...") self.logger.debug(str(data)) self.influxdb_client.write_points([data]) except Exception as e: self.logger.error("Error " + str(e)) def run(self): with concurrent.futures.ThreadPoolExecutor( max_workers=100) as executor: executor.submit(self.send_data) containers = self.docker_client.containers() collectors = {} for container in containers: collectors[container["Id"]] = executor.submit( self.collecting, container["Id"]) events = self.docker_client.events(decode=True) for event in events: if event["status"] == "start": self.logger.info("Start collector for: " + event["id"]) collectors[event["id"]] = executor.submit( self.collecting, event["id"]) elif event["status"] == "die": try: self.logger.info("Cancel collector for: " + event["id"]) collectors[event["id"]].cancel() except Exception as e: self.logger.debug( "Exception when Cancel collector: {} : {}".format( str(e), event["id"]))
def status(conf, config): """Get the status of the running containers""" config = open_config(config) docker_client = Client('unix://var/run/docker.sock') all_containers = docker_client.containers(all=True) running_containers = [] running_ids = dict() names = get_config_names(config) for container in all_containers: name = container['Names'][0].replace('/', '') if name in names: print_cont = dict() print_cont['name'] = name print_cont['status'] = container['Status'] print_cont['image'] = container['Image'] print_cont['image_id'] = container['ImageID'] running_ids[container['ImageID']] = print_cont['image'] print_cont['ports'] = [] if 'Ports' in container: # pragma: no cover for port in container['Ports']: pub_port = None priv_port = None if 'PublicPort' in port: pub_port = port['PublicPort'] if 'PrivatePort' in port: priv_port = port['PrivatePort'] if pub_port: print_cont['ports'] += [ '{}:{}'.format(pub_port, priv_port) ] running_containers.append(print_cont) else: # pragma: no cover click.echo( click.style(a_text('{}'.format(name), 'not in the config'), fg=col_warn)) click.echo( click.style('Running containers'.center(80, '='), fg=col_info, bold=True)) click.echo() for container in running_containers: click.echo( click.style('{}'.format(container['name']).center(80, '-'), fg=col_info, bold=True)) for k in container: if isinstance(container[k], list): container[k] = ' '.join(container[k]) if len(container[k]) > 40: cut = len(container[k]) - 40 container[k] = container[k][:cut - 3] + '...' click.echo(click.style(a_text(k, container[k]), fg=col_info)) click.echo('\n') images = docker_client.images() click.echo( click.style('Images from the config'.center(80, '='), fg=col_info, bold=True)) click.echo() for image in images: if image['Id'] in running_ids: print_im = dict() print_im['name'] = '{}'.format(running_ids[image['Id']]) print_im['created'] = pd.to_datetime(image['Created'] * 1e9) print_im['created'] = print_im['created'].strftime( '%Y-%m-%d %H:%M') print_im['size'] = '{:.2f}'.format(image['Size'] / 1000000000.) click.echo( click.style('{}'.format(print_im['name']).center(80, '-'), fg=col_info, bold=True)) for k in print_im: if isinstance(print_im[k], list): # pragma: no cover container[k] = ' '.join(print_im[k]) if len(print_im[k]) > 40: # pragma: no cover cut = len(print_im[k]) - 40 container[k] = print_im[k][:cut - 3] + '...' click.echo(click.style(a_text(k, print_im[k]), fg=col_info)) click.echo('\n') return 0
class DockerClient(object): """ Talk to the real Docker server directly. Some operations can take a while (e.g. stopping a container), so we use a thread pool. See https://github.com/ClusterHQ/flocker/issues/718 for using a custom thread pool. :ivar unicode namespace: A namespace prefix to add to container names so we don't clobber other applications interacting with Docker. """ def __init__(self, namespace=BASE_NAMESPACE): self.namespace = namespace self._client = Client(version="1.12") def _to_container_name(self, unit_name): """ Add the namespace to the container name. :param unicode unit_name: The unit's name. :return unicode: The container's name. """ return self.namespace + unit_name def add(self, unit_name, image_name, ports=None, environment=None, volumes=()): container_name = self._to_container_name(unit_name) if environment is not None: environment = environment.to_dict() if ports is None: ports = [] def _create(): self._client.create_container( image_name, name=container_name, environment=environment, volumes=list(volume.container_path.path for volume in volumes), ports=[p.internal_port for p in ports]) def _add(): try: _create() except APIError as e: if e.response.status_code == NOT_FOUND: # Image was not found, so we need to pull it first: self._client.pull(image_name) _create() else: raise # Just because we got a response doesn't mean Docker has # actually updated any internal state yet! So if e.g. we did a # stop on this container Docker might well complain it knows # not the container of which we speak. To prevent this we poll # until it does exist. while not self._blocking_exists(container_name): sleep(0.001) continue self._client.start(container_name, binds={ volume.node_path.path: { u"bind": volume.container_path.path, u"ro": False } for volume in volumes }, port_bindings={ p.internal_port: p.external_port for p in ports }) d = deferToThread(_add) def _extract_error(failure): failure.trap(APIError) code = failure.value.response.status_code if code == 409: raise AlreadyExists(unit_name) return failure d.addErrback(_extract_error) return d def _blocking_exists(self, container_name): """ Blocking API to check if container exists. :param unicode container_name: The name of the container whose existence we're checking. :return: ``True`` if unit exists, otherwise ``False``. """ try: self._client.inspect_container(container_name) return True except APIError: return False def exists(self, unit_name): container_name = self._to_container_name(unit_name) return deferToThread(self._blocking_exists, container_name) def remove(self, unit_name): container_name = self._to_container_name(unit_name) def _remove(): try: self._client.stop(container_name) self._client.remove_container(container_name) except APIError as e: # 500 error code is used for "this was already stopped" in # older versions of Docker. Newer versions of Docker API # give NOT_MODIFIED instead, so we can fix this when we # upgrade: https://github.com/ClusterHQ/flocker/issues/721 if e.response.status_code in (NOT_FOUND, INTERNAL_SERVER_ERROR): return # Can't figure out how to get test coverage for this, but # it's definitely necessary: raise d = deferToThread(_remove) return d def list(self): def _list(): result = set() ids = [ d[u"Id"] for d in self._client.containers(quiet=True, all=True) ] for i in ids: data = self._client.inspect_container(i) state = (u"active" if data[u"State"][u"Running"] else u"inactive") name = data[u"Name"] if name.startswith(u"/" + self.namespace): name = name[1 + len(self.namespace):] else: continue # We'll add missing info in # https://github.com/ClusterHQ/flocker/issues/207 # and to extract volume info from the inspect results: # https://github.com/ClusterHQ/flocker/issues/289 result.add( Unit(name=name, container_name=self._to_container_name(name), activation_state=state, container_image=None)) return result return deferToThread(_list)
import os import subprocess import pystache import urllib2 from docker import Client from string import Template c = Client(base_url='unix://mount/run/docker.sock') f = open('/config.tpl', 'r') v = { 'Memory': [], 'CarbonHost': os.environ['CARBON_PORT_2003_TCP_ADDR'], 'CarbonPort': os.environ['CARBON_PORT_2003_TCP_PORT'] } for val in c.containers(): v['Memory'].append({'Id': val['Id'], 'Instance': val['Names'][0]}) config = pystache.render(f.read(), v) output = open('/etc/collectd/collectd.conf', 'w') output.write(config) output.close() subprocess.check_call(["/usr/sbin/collectd", "-f"])
class DockerManager: """ Docker Manager Object Model """ def __init__(self, path=None, url=None): self._url = url or CONF.clients_docker.url self.dockerfile_path = path or CONF.clients_docker.build_dir self.dc = DC(base_url=self._url) def list_images(self): """ List current supported systems :return: """ systems = [] LOG.info("Searching supported systems in %s" % self.dockerfile_path) for df in os.listdir(self.dockerfile_path): image_path = os.path.join(self.dockerfile_path, df) system = df.split("-")[0] if os.path.splitext(df)[1] == ".dockerfile": with open(image_path) as dff: cont = dff.read() m = re.search("FROM ([^:]+):([^\n]+)\n", cont) if m: tag = re.findall("(?im)^# tag: (.*)$", cont)[0].strip() systems.append({ 'name': m.group(1), 'version': m.group(2), 'dockerfile': image_path, 'system': system, 'tag': tag }) return systems def generate_image(self, df): """generate docker image""" status = True with open(df) as dockerfile: tag = re.findall("(?im)^# tag: (.*)$", dockerfile.read())[0].strip() LOG.info("Generating %s from %s" % (tag, df)) if tag: resp = self.dc.build(path=CONF.clients_docker.build_dir, dockerfile=df, rm=True, tag=tag) for l in resp: if "error" in l.lower(): status = False LOG.debug(l) else: status = False return status def download_image(self, tag): status = True LOG.info("Downloading image %s" % tag) try: res = self.dc.pull(tag) for l in res: LOG.debug(l) except Exception as e: status = False import traceback traceback.print_exc() return status def prepare_image(self, tag): """ Generate image from local file or download if necessary :param tag: image tag :return: operation status """ status = True LOG.info("Preparing Image %s" % tag) available_images = [ t['RepoTags'][0].split(":")[0] for t in self.dc.images() ] if tag not in available_images: status = self.download_image(tag) if not status: df = [ d['dockerfile'] for d in self.list_images() if d['tag'] == tag ][0] status = self.generate_image(df) return status def run_container(self, user, cookbook, image_name): """Run and start a container based on the given image mounts local cookbook storage path :param image_name: image to run :return: operation status """ status = True error = "Image successfully created: [%s]" % image_name self.prepare_image(image_name) contname = self.generate_container_name(user, cookbook, image_name) try: self.remove_container(contname, kill=True) container = self.dc.create_container( image_name, tty=True, volumes=[CONF.clients_git.repo_path], name=contname, host_config=self.dc.create_host_config(binds={ CONF.clients_git.repo_path: { 'bind': CONF.clients_git.repo_path, 'mode': 'rw', }, }, network_mode="host"), ).get('Id') self.dc.start(container=container) except NotFound as e: error = "Image not found: [%s]" % image_name LOG.error(_LW(error)) status = False except AttributeError as e: error = "Error creating container: [%s]" % e LOG.error(_LW(error)) status = False # raise DockerContainerException(image=image_name) return status, error def remove_container(self, contname, kill=True): """destroy container on exit :param kill: inhibits removal for testing purposes """ found = self.get_container_by_name(contname) if found: LOG.info(_LI('Removing old container [%s]' % contname)) if kill: self.dc.remove_container(found, force=True) else: self.dc.stop(found) def execute_command(self, contname, command): """ Execute a command in the given container :param command: bash command to run :return: execution result """ cont = self.get_container_by_name(contname) bash_txt = "/bin/bash -c \"{}\"".format(command.replace('"', '\\"')) exec_txt = self.dc.exec_create(container=cont, cmd=bash_txt) return self.dc.exec_start(exec_txt) def get_container_by_name(self, contname): """ :param contname: name or alias of the container to find :return: container found """ found = None for c in self.dc.containers(all=True): for n in c['Names']: if contname in n: found = c break if not found: LOG.info(_LI('Container not found for removal [%s]' % contname)) return found def generate_container_name(self, user, cookbook, image_name): """ :param user: user name :param cookbook: cookbook name :param image_name: docker image name :return: regular generated container name """ return "{}_{}_{}-validate".format(self.generate_user_name(user), cookbook, image_name.replace("/", "_")).lower() def generate_user_name(self, user): """ :param user: input string :return: cleaned up username """ return "".join(c for c in user if c.isalnum() or c in ".-_")
class AppDocker(object): def __init__(self): self.client = Client() def create_start(self, app): app_name = app.name container_name = CONTAINER_NAME_PREFIX + app_name create_args = { 'image': CONTAINER_IMAGE, 'tty': False, 'detach': True, 'mem_limit': settings.CONTAINER_MEM_LIMIT, 'name': container_name, } container = self.client.create_container(**create_args) container_id = container.get('Id') if not container_id: raise AppDockerException("Failed to create container") host_control_path = app.host_control_path host_app_path = app.host_app_path bind_volumes = { host_control_path: { 'bind': settings.CONTAINER_CONTROL_DIR, 'ro': False, }, host_app_path: { 'bind': settings.CONTAINER_APP_DIR, 'ro': False, }, } start_args = { 'container': container_id, 'binds': bind_volumes, } self.client.start(**start_args) return container_id def stop(self, app): container_id = app.container_id stop_args = { 'container': container_id, 'timeout': STOP_TIMEOUT, } self.client.stop(**stop_args) def start(self, app): container_id = app.container_id start_args = { 'container': container_id, } self.client.start(**start_args) def remove(self, app): container_id = app.container_id remove_args = { 'container': container_id, 'force': True, } self.client.remove_container(**remove_args) def exists(self, app): container_id = app.container_id for container in self.client.containers(): if container.get('Id') == container_id: return True return False
class Docker(object): def __init__(self, socket, remove_intermediate=False, registry=None, insecure_registry=False): log.debug('DockerClient connecting to {}'.format(socket)) self._socket = socket self._client = Client(base_url=socket, timeout=300, version='auto') self._remove_intermediate = remove_intermediate self._registry = registry self._insecure_registry = insecure_registry @exception_safe(ConnectionError, False) def build(self, dockerfile, image): """/ :param dockerfile: The dockerfile full path :param image: the image name (eg: midolman:1.9) """ log.info('Now building {}'.format(image)) log.debug('Invoking docker build on {}'.format(dockerfile)) response = self._client.build(path=os.path.dirname(dockerfile), tag=image, pull=False, rm=self._remove_intermediate, dockerfile=os.path.basename(dockerfile)) last_line = None for line in response: eval_line = eval(line) if 'stream' in eval_line: print(eval_line['stream']), last_line = eval_line['stream'] # Check for ensure success after building if 'Successfully built' not in last_line: log.error('Error building the image {}.'.format(image)) return False return True @exception_safe(ConnectionError, None) def pull(self, image): """ :param image: The image name with its tag :param repository: The repository where to pull the image (dockerhub if none defined) """ name, tag = image.split(':') repository = '{}/{}'.format(self._registry, name) \ if self._registry else name log.info('Now Pulling {}:{}'.format(repository, tag)) response = self._client.pull(repository=repository, tag=tag, insecure_registry=self._insecure_registry, stream=True) for line in response: eval_line = eval(line) if 'error' in eval_line: log.error( 'Error pulling image: {}'.format(eval_line['error'])) return False if 'status' in eval_line: log.info('[{}:{}] Status: {}'.format(repository, tag, eval_line['status'])) if self._registry: # If pulling from an external repo, we need to tag with the # actual image name used in the flavours definition. images = self.list_images(repository) for image in images: for repotag in image['RepoTags']: if '{}:{}'.format(repository, tag) == repotag: self._client.tag(image['Id'], name, tag, force=True) return True @exception_safe(ConnectionError, None) def push(self, image): """ :param image: The image name with its tag :param repository: The repository where to push the image (dockerhub if none defined) """ name, tag = image.split(':') if self._registry: # First tag the local image to map to the new registry repository = '{}/{}'.format(self._registry, name) images = self.list_images(name) for image in images: for repotag in image['RepoTags']: if '{}:{}'.format(name, tag) == repotag: self._client.tag(image['Id'], repository, tag, force=True) else: repository = name log.info('Now pushing {}:{}'.format(repository, tag)) response = self._client.push(repository=repository, tag=tag, insecure_registry=self._insecure_registry, stream=True) for line in response: eval_line = eval(line) if 'error' in eval_line: log.error( 'Error pushing image: {}'.format(eval_line['error']) ) return False if 'status' in eval_line: if 'Pushing' not in eval_line['status'] \ and 'Buffering' not in eval_line['status']: log.info('[{}:{}] Status: {}'.format(repository, tag, eval_line['status'])) return True @exception_safe(ConnectionError, []) def list_images(self, prefix=None): """ List the available images :param prefix: Filter the images by a prefix (eg: "sandbox/") :return: the images list """ images = self._client.images() if prefix: filtered = list() for image in images: if 'RepoTags' in image and image['RepoTags'] is not None: for tag in image['RepoTags']: if tag.startswith(prefix): filtered.append(image) images = filtered return images def list_containers(self, prefix=None): """ List the running containers, prefixed with prefix :param prefix: The container's name prefix :return: The list of containers """ containers = self._client.containers() filtered = list() if prefix: for container_ref in containers: if prefix in container_ref['Names'][0]: filtered.append(container_ref) containers = filtered return containers def container_by_name(self, name): containers = self.list_containers() for container_ref in containers: if name == self.principal_container_name(container_ref): return container_ref @staticmethod def principal_container_name(container_ref): for name in container_ref['Names']: if '/' not in name[1:]: return name[1:] def container_ip(self, container_ref): return self._client.inspect_container( container_ref )['NetworkSettings']['IPAddress'] def stop_container(self, container_ref): self._client.stop(container_ref) def kill_container(self, container_ref): self._client.kill(container_ref) def remove_container(self, container_ref): self._client.remove_container(container_ref) @exception_safe(OSError, False) def execute(self, container_ref, command): """ Execute a command inside the container. NOTE: Needs the 'docker' binary installed in the host """ cmd = ['docker', 'exec', '-it', self.principal_container_name(container_ref), 'env', 'TERM=xterm', command] log.debug('Running command: "{}"'.format(' '.join(cmd))) p = subprocess.Popen(cmd, stderr=subprocess.STDOUT) p.wait() def ssh(self, container_ref): self.execute(container_ref, 'bash')
from docker import APIClient as Client from nsenter import Namespace from fabric.api import local TEST_BASE_DIR = '/tmp/goplane' IN6_ADDR_GEN_MODE_NONE = 1 dckr = Client() flatten = lambda l: itertools.chain.from_iterable(l) random_str = lambda n: ''.join( [random.choice(string.ascii_letters + string.digits) for i in range(n)]) get_containers = lambda: [ str(x)[1:] for x in list(flatten(n['Names'] for n in dckr.containers(all=True))) ] class docker_netns(object): def __init__(self, name): pid = int(dckr.inspect_container(name)['State']['Pid']) if pid == 0: raise Exception('no container named {0}'.format(name)) self.pid = pid def __enter__(self): pid = self.pid if not os.path.exists('/var/run/netns'): os.mkdir('/var/run/netns') os.symlink('/proc/{0}/ns/net'.format(pid),