def _init_cli(self): if os.path.exists('/var/run/docker.sock'): self.cli = APIClient(base_url='unix://var/run/docker.sock', version=self.api_version) else: kwargs = kwargs_from_env(assert_hostname=False) kwargs['version'] = self.api_version self.cli = APIClient(**kwargs)
def worklist(): l = sorted( list( map( lambda conr: APIClient().inspect_container(conr.name)["State"][ "Pid"], from_env().containers.list()))) l.remove(APIClient().inspect_container("dborch")["State"]["Pid"]) l.remove(APIClient().inspect_container("ubuntu_rmq_1")["State"]["Pid"]) l.remove(APIClient().inspect_container("ubuntu_zoo_1")["State"]["Pid"]) return jsonify(l), 200
def __init__(self, url, name): self.url = url self.name = name self.cli = APIClient(base_url=url) self.bwb_instance_id = socket.gethostname() self.bwbMounts={} self.findVolumeMappings()
def __init__(self, url, name): self.url = url self.name = name self.cli = APIClient(base_url=url) outputString = str( subprocess.check_output( 'cat /proc/self/cgroup | grep devices | head -1 | cut -d "/" -f3 | sed "s/.*-//g" | sed "s/\..*//g"', shell=True, universal_newlines=True, )) if outputString: self.bwb_instance_id = outputString.splitlines()[0] else: outputString = str( subprocess.check_output( "cat /proc/self/mountinfo | grep -oP '(?<=containers/).*?(?=/resolv)'", shell=True, universal_newlines=True, )) self.bwb_instance_id = outputString.strip() print(self.bwb_instance_id) self.bwbMounts = {} self.shareMountPoint = {} self.findVolumeMappings() self.findShareMountPoint(overwrite=True) #self.findShareMountPoint() self.logFile = None self.schedulerStarted = False
def setUp(self): cli = APIClient() try: cli.remove_container("eth") except: pass self.client = lib.parity.Parity("Ropsten", "light")
def build_delay(task: int, host, build_type, tag, admin, pt=None, dockerfile=None): """ 编译镜像 """ task = db.session.query(TaskList).get(task) instance = db.session.query(Host).filter(Host.id == host).one_or_none() cli = APIClient(base_url='tcp://{}'.format(instance.addr)) if build_type == 'tar': f = open(pt, 'rb') for line in cli.build(fileobj=f, rm=True, tag=tag, custom_context=True): print(line) task_add_log(task.id, line) task.status = task.STATUS_DONE elif build_type == 'pull': for line in cli.pull(tag, stream=True, decode=True): task_add_log(task.id, line) task.status = task.STATUS_DONE else: try: f = BytesIO(dockerfile.encode('utf-8')) for line in cli.build(fileobj=f, rm=True, tag=tag): task_add_log(task.id, line) task.status = task.STATUS_DONE except docker_error.DockerException as e: task.status = task.STATUS_ERROR task.remark = str(e) db.session.commit()
def publish(self, img): if self.docker_client is None: self.docker_client = APIClient(version='auto') # TODO: do we need to set tag? for line in self.docker_client.push(img, stream=True): self._process_stream(line)
def get_client(self): """Create Docker client connection.""" try: if self.client is None: self.client = APIClient( base_url=self.docker_socket, version=DOCKER_VERSION, ) return self.client except DockerException: log.exception( LOG_TEMPLATE, { 'project': self.project.slug, 'version': self.version.slug, 'msg': "Could not connect to Docker API", } ) # We don't raise an error here mentioning Docker, that is a # technical detail that the user can't resolve on their own. # Instead, give the user a generic failure raise BuildEnvironmentError( BuildEnvironmentError.GENERIC_WITH_BUILD_ID.format( build_id=self.build['id'], ), )
def __init__(self, repo_path, from_image, image_name, image_tag, copy_code=True, in_tmp_repo=True, build_steps=None, env_vars=None, dockerfile_name='Dockerfile'): # This will help create a unique tmp folder for dockerizer in case of concurrent jobs self.uuid = uuid.uuid4().hex self.from_image = from_image self.image_name = image_name self.image_tag = image_tag self.repo_path = repo_path self.folder_name = repo_path.split('/')[-1] self.copy_code = copy_code self.in_tmp_repo = in_tmp_repo if in_tmp_repo and copy_code: self.build_repo_path = self.create_tmp_repo() else: self.build_repo_path = self.repo_path self.build_path = '/'.join(self.build_repo_path.split('/')[:-1]) self.build_steps = get_list(build_steps) self.env_vars = get_list(env_vars) self.dockerfile_path = os.path.join(self.build_path, dockerfile_name) self.polyaxon_requirements_path = self._get_requirements_path() self.polyaxon_setup_path = self._get_setup_path() self.docker = APIClient(version='auto') self.registry_host = None self.docker_url = None
def delete_image(args): """ - API to remove Image. - API is equivalent to docker rmi cmd. - Response is in dict: Format: {'force':'force','image':'Image_name'} Returns 'Success' if container is removed successfully if not 'Failed'. """ if 'image' in args: args['image'] = str(args['image']) if 'force' in args: args['force'] = bool(args['force']) if 'noprune' in args: args['noprune'] = bool(args['noprune']) invoke_clientAPI = APIClient(base_url='unix://var/run/docker.sock', version='auto') status = {} try: image_removed = invoke_clientAPI.remove_image(**args) status["status"] = "True" except: status["status"] = "False" return json.dumps(status)
def list_containers(args): """ - API to list all running containers. - API is equivalent to docker ps command. - Response is a list of dictionary: [{u'Status': u'Up 25 minutes', u'Created': 1477874345, u'Image': u'sha256:2b786d1d393fca95d9baa72c40b7d2da8b4fc3135659b7ca3046967f8de09c15', u'Labels': {}, u'NetworkSettings': {u'Networks': {u'bridge': {u'NetworkID': u'f3211da5394d90c58365fb9f50285480735171ef88a4f21399ec08575797f21f', u'MacAddress': u'02:42:ac:11:00:02', u'GlobalIPv6PrefixLen': 0, u'Links': None, u'GlobalIPv6Address': u'', u'IPv6Gateway': u'', u'IPAMConfig': None, u'EndpointID': u'932507e69777dc1c9bd5784941d92722889a9d76af1a10672afb1c063c092398', u'IPPrefixLen': 16, u'IPAddress': u'172.17.0.2', u'Gateway': u'172.17.0.1', u'Aliases': None}}}, u'HostConfig': {u'NetworkMode': u'default'}, u'ImageID': u'sha256:2b786d1d393fca95d9baa72c40b7d2da8b4fc3135659b7ca3046967f8de09c15', u'State': u'running', u'Command': u'/bin/bash', u'Names': [u'/nostalgic_bhabha'], u'Mounts': [], u'Id': u'2b5c2b8de6610c2d443518a84ca7c56ded98fbcf9a70c02ea73746b5c05dd21e', u'Ports': []}] """ if 'all' in args: args['all'] = bool(args['all']) invoke_clientAPI = APIClient(base_url='unix://var/run/docker.sock', version='auto') container_list = invoke_clientAPI.containers(**args) return container_list
def execute(self, context): logging.info('Starting docker container from image ' + self.image) tls_config = None if self.tls_ca_cert and self.tls_client_cert and self.tls_client_key: tls_config = tls.TLSConfig(ca_cert=self.tls_ca_cert, client_cert=(self.tls_client_cert, self.tls_client_key), verify=True, ssl_version=self.tls_ssl_version, assert_hostname=self.tls_hostname) self.docker_url = self.docker_url.replace('tcp://', 'https://') self.cli = APIClient(base_url=self.docker_url, version=self.api_version, tls=tls_config) if ':' not in self.image: image = self.image + ':latest' else: image = self.image if self.force_pull or len(self.cli.images(name=image)) == 0: logging.info('Pulling docker image ' + image) for l in self.cli.pull(image, stream=True): output = json.loads(l.decode('utf-8')) logging.info("{}".format(output['status'])) cpu_shares = int(round(self.cpus * 1024)) with TemporaryDirectory(prefix='airflowtmp') as host_tmp_dir: self.environment['AIRFLOW_TMP_DIR'] = self.tmp_dir self.volumes.append('{0}:{1}'.format(host_tmp_dir, self.tmp_dir)) self.container = self.cli.create_container( command=self.get_command(), cpu_shares=cpu_shares, environment=self.environment, host_config=self.cli.create_host_config( binds=self.volumes, network_mode=self.network_mode), image=image, mem_limit=self.mem_limit, user=self.user) self.cli.start(self.container['Id']) line = '' for line in self.cli.logs(container=self.container['Id'], stream=True): line = line.strip() if hasattr(line, 'decode'): line = line.decode('utf-8') logging.info(line) exit_code = self.cli.wait(self.container['Id']) if exit_code != 0: raise AirflowException('docker container failed') if self.xcom_push: return self.cli.logs(container=self.container['Id'] ) if self.xcom_all else str(line)
def delete_container(args): """ - API to remove container - API is equivalent to docker rm cmd - Response is in dict: Format: {'container': 'container_name'} Returns 'Success' if container is deleted successfully if not 'Failed' """ if 'container' in args: args['container'] = str(args['container']) if 'v' in args: args['v'] = bool(args['v']) if 'link' in args: args['link'] = bool(args['link']) if 'force' in args: args['force'] = bool(args['force']) invoke_clientAPI = APIClient(base_url='unix://var/run/docker.sock', version='auto') status = {} try: container_removed = invoke_clientAPI.remove_container(**args) status["status"] = "True" except: status["status"] = "False" return json.dumps(status)
def docker_abs_net_io(container_id): """ Network traffic of all network interfaces within the controller. :param container_id: The full ID of the docker container. :type container_id: ``str`` :return: Returns the absolute network I/O till container startup, in bytes. The return dict also contains the system time. :rtype: ``dict`` """ c = APIClient() command = c.exec_create(container_id, 'ifconfig') ifconfig = c.exec_start(command['Id']) sys_time = int(time.time() * 1000000000) in_bytes = 0 m = re.findall('RX bytes:(\d+)', str(ifconfig)) if m: for number in m: in_bytes += int(number) else: in_bytes = None out_bytes = 0 m = re.findall('TX bytes:(\d+)', str(ifconfig)) if m: for number in m: out_bytes += int(number) else: out_bytes = None return {'NET_in': in_bytes, 'NET_out': out_bytes, 'NET_systime': sys_time}
def __init__(self, build_job: 'BuildJob', repo_path: str, from_image: str, copy_code: bool = True, build_steps: Optional[List[str]] = None, env_vars: Optional[List[Tuple[str, str]]] = None, dockerfile_name: str = 'Dockerfile') -> None: self.build_job = build_job self.job_uuid = build_job.uuid.hex self.job_name = build_job.unique_name self.from_image = from_image self.image_name = get_image_name(self.build_job) self.image_tag = self.job_uuid self.folder_name = repo_path.split('/')[-1] self.repo_path = repo_path self.copy_code = copy_code self.build_path = '/'.join(self.repo_path.split('/')[:-1]) self.build_steps = to_list(build_steps, check_none=True) self.env_vars = to_list(env_vars, check_none=True) self.dockerfile_path = os.path.join(self.build_path, dockerfile_name) self.polyaxon_requirements_path = self._get_requirements_path() self.polyaxon_setup_path = self._get_setup_path() self.docker = APIClient(version='auto') self.registry_host = None self.docker_url = None self.is_pushing = False
def run(self, img, name, options): if self.docker_client is None: self.docker_client = APIClient(version='auto') ports = [] host_config = None # Prepare port configuration if options.ports is not None and len(options.ports) > 0: for port_number in options.ports: ports.append(port_number) host_config = self.docker_client.create_host_config( port_bindings={p: p for p in ports}) # Launch docker container container = self.docker_client.create_container( img, name=name, ports=ports, host_config=host_config, ) self.docker_client.start(container=container.get('Id')) self.container = container logger.info('Starting container {}'.format(container))
def execute(self, context): tls_config = self.__get_tls_config() if self.docker_conn_id: self.cli = self.get_hook().get_conn() else: self.cli = APIClient( base_url=self.docker_url, version=self.api_version, tls=tls_config ) # Pull the docker image if `force_pull` is set or image does not exist locally if self.force_pull or len(self.cli.images(name=self.image)) == 0: self.log.info('Pulling docker image %s', self.image) for output in self.cli.pull(self.image, stream=True, decode=True): if isinstance(output, six.string_types): self.log.info("%s", output) if isinstance(output, dict) and 'status' in output: self.log.info("%s", output['status']) self.environment['AIRFLOW_TMP_DIR'] = self.tmp_dir return self._run_image()
def __init__(self, info, dockerfile, repository, tag, to_build, stage, context=None, to_push=True, additional_tags=[], target=None): # Meta-data about the image should go to info. # All keys in info are accessible as attributes # of this class self.info = info self.summary = {} self.build_args = {} self.labels = {} self.stage = stage self.dockerfile = dockerfile self.context = context self.to_push = to_push # TODO: Add ability to tag image with multiple tags self.repository = repository self.tag = tag self.additional_tags = additional_tags self.ecr_url = f"{self.repository}:{self.tag}" if not isinstance(to_build, bool): to_build = True if to_build == "true" else False self.to_build = to_build self.build_status = None self.client = APIClient(base_url=constants.DOCKER_URL, timeout=constants.API_CLIENT_TIMEOUT) self.log = [] self._corresponding_common_stage_image = None self.target = target
def docker_client(): """ Returns a docker-py client configured using environment variables according to the same logic as the official Docker client. """ cert_path = os.environ.get('DOCKER_CERT_PATH', '') if cert_path == '': cert_path = os.path.join(os.environ.get('HOME', ''), '.docker') base_url = os.environ.get('DOCKER_HOST') tls_config = None if os.environ.get('DOCKER_TLS_VERIFY', '') != '': parts = base_url.split('://', 1) base_url = '%s://%s' % ('https', parts[1]) client_cert = (os.path.join(cert_path, 'cert.pem'), os.path.join(cert_path, 'key.pem')) ca_cert = os.path.join(cert_path, 'ca.pem') tls_config = tls.TLSConfig( ssl_version=ssl.PROTOCOL_TLSv1, verify=True, assert_hostname=False, client_cert=client_cert, ca_cert=ca_cert, ) timeout = int(os.environ.get('DOCKER_CLIENT_TIMEOUT', 60)) return APIClient(base_url=base_url, tls=tls_config, version='1.21', timeout=timeout)
def docker_client(environment, version=None, tls_config=None, host=None, tls_version=None): """ Returns a docker-py client configured using environment variables according to the same logic as the official Docker client. """ try: kwargs = kwargs_from_env(environment=environment, ssl_version=tls_version) except TLSParameterError: raise UserError( "TLS configuration is invalid - make sure your DOCKER_TLS_VERIFY " "and DOCKER_CERT_PATH are set correctly.\n" "You might need to run `eval \"$(docker-machine env default)\"`") if host: kwargs['base_url'] = host if tls_config: kwargs['tls'] = tls_config if version: kwargs['version'] = version timeout = environment.get('COMPOSE_HTTP_TIMEOUT') if timeout: kwargs['timeout'] = int(timeout) else: kwargs['timeout'] = HTTP_TIMEOUT kwargs['user_agent'] = generate_user_agent() return APIClient(**kwargs)
def __init__(self, tc_command_output=TcCommandOutput.NOT_SET): self.__client = APIClient(version="auto") self.__host_name = os.uname()[1] self.__tc_command_output = tc_command_output self.__con = connect_memdb() IfIndex.attach(self.__con)
def execute(self, context): '''Modified only to use the get_host_tmp_dir helper.''' self.log.info('Starting docker container from image %s', self.image) tls_config = self.__get_tls_config() if self.docker_conn_id: self.cli = self.get_hook().get_conn() else: self.cli = APIClient(base_url=self.docker_url, version=self.api_version, tls=tls_config) if self.force_pull or len(self.cli.images(name=self.image)) == 0: self.log.info('Pulling docker image %s', self.image) for l in self.cli.pull(self.image, stream=True): output = seven.json.loads(l.decode('utf-8').strip()) if 'status' in output: self.log.info("%s", output['status']) with self.get_host_tmp_dir() as host_tmp_dir: self.environment['AIRFLOW_TMP_DIR'] = self.tmp_dir self.volumes.append('{0}:{1}'.format(host_tmp_dir, self.tmp_dir)) self.container = self.cli.create_container( command=self.get_command(), environment=self.environment, host_config=self.cli.create_host_config( auto_remove=self.auto_remove, binds=self.volumes, network_mode=self.network_mode, shm_size=self.shm_size, dns=self.dns, dns_search=self.dns_search, cpu_shares=int(round(self.cpus * 1024)), mem_limit=self.mem_limit, ), image=self.image, user=self.user, working_dir=self.working_dir, ) self.cli.start(self.container['Id']) res = [] line = '' for new_line in self.cli.logs(container=self.container['Id'], stream=True): line = new_line.strip() if hasattr(line, 'decode'): line = line.decode('utf-8') self.log.info(line) res.append(line) result = self.cli.wait(self.container['Id']) if result['StatusCode'] != 0: raise AirflowException( 'docker container failed with result: {result} and logs: {logs}'.format( result=repr(result), logs='\n'.join(res) ) ) if self.xcom_push_flag: # Try to avoid any kind of race condition? return res if self.xcom_all else str(line)
def execute(self, context): self.log.info('Starting docker container from image %s', self.image) tls_config = self.__get_tls_config() if self.docker_conn_id: self.cli = self.get_hook().get_conn() else: self.cli = APIClient(base_url=self.docker_url, version=self.api_version, tls=tls_config) if self.force_pull or len(self.cli.images(name=self.image)) == 0: self.log.info('Pulling docker image %s', self.image) for l in self.cli.pull(self.image, stream=True): output = json.loads(l.decode('utf-8').strip()) if 'status' in output: self.log.info("%s", output['status']) with TemporaryDirectory(prefix='airflowtmp', dir=self.host_tmp_dir) as host_tmp_dir: self.environment['AIRFLOW_TMP_DIR'] = self.tmp_dir self.volumes.append('{0}:{1}'.format(host_tmp_dir, self.tmp_dir)) self.container = self.cli.create_container( command=self.get_command(), environment=self.environment, host_config=self.cli.create_host_config( auto_remove=self.auto_remove, binds=self.volumes, network_mode=self.network_mode, shm_size=self.shm_size, dns=self.dns, dns_search=self.dns_search, cpu_shares=int(round(self.cpus * 1024)), mem_limit=self.mem_limit), image=self.image, user=self.user, working_dir=self.working_dir) self.cli.start(self.container['Id']) line = '' for line in self.cli.attach(container=self.container['Id'], stdout=True, stderr=True, stream=True): line = line.strip() if hasattr(line, 'decode'): line = line.decode('utf-8') self.log.info(line) result = self.cli.wait(self.container['Id']) if result['StatusCode'] != 0: raise AirflowException('docker container failed: ' + repr(result)) # duplicated conditional logic because of expensive operation if self.do_xcom_push: return self.cli.logs(container=self.container['Id']) \ if self.xcom_all else line.encode('utf-8')
def container_peripherelAccess(self, **kwargs): """ - API creates container and also provides peripherel access. - API is equivalent to create container with host configurations added to it. - Response """ host_config = {} # image = kwargs['image'] # network_disabled = kwargs['network_disabled'] # host_config = {'devices': '/sys/class/leds:/:rwm'} # print image,host_config invoke_clientAPI = APIClient(base_url='unix://var/run/docker.sock', version='auto') containerID = invoke_clientAPI.create_container( 'ubuntu', 'true', stdin_open=bool('True'), command=list['/bin/bash'], host_config=invoke_clientAPI.create_host_config( devices=['/dev/sda:rwm'])) # containerID = invoke_clientAPI.create_container(image) return containerID
def get_conn(self) -> APIClient: client = APIClient(base_url=self.__base_url, version=self.__version, tls=self.__tls, timeout=self.__timeout) self.__login(client) return client
def run_command_in_container(container_image, command): client = APIClient() # Create the container container = client.create_container( image=container_image, command=('/bin/sh -c "sleep {time}; exit {exit}"'.format( time=60, exit=42, )), detach=True, user='******', ) # Start the container container_id = container.get('Id') client.start(container=container_id) # Execute commands to verify versions exec_cmd = client.exec_create( container=container_id, cmd=f"/bin/sh -c '{command}'", stdout=True, stderr=True, ) cmd_output = client.exec_start(exec_id=exec_cmd['Id'], stream=False) cmd_output = cmd_output.decode('utf-8', 'replace').strip() # Kill container client.kill(container_id) return cmd_output
def replicateContainer(containerID, orig_url, dest_url): orig_client = APIClient(base_url=orig_url, tlf=False) dest_client = APIClient(base_url=dest_url, tls=False) image = orig_client.get_image(containerID) data=None for chunck in image: if data == None: data = chunck else: data += chunck dest_client.load_image(data) container = dest_client.create_container(containerID, name=containerID)
def docker_client(environment, version=None, context=None, tls_version=None): """ Returns a docker-py client configured using environment variables according to the same logic as the official Docker client. """ try: kwargs = kwargs_from_env(environment=environment, ssl_version=tls_version) except TLSParameterError: raise UserError( "TLS configuration is invalid - make sure your DOCKER_TLS_VERIFY " "and DOCKER_CERT_PATH are set correctly.\n" "You might need to run `eval \"$(docker-machine env default)\"`") if not context: # check env for DOCKER_HOST and certs path host = kwargs.get("base_url", None) tls = kwargs.get("tls", None) verify = False if not tls else tls.verify if host: context = Context("compose", host=host, tls=verify) else: context = ContextAPI.get_current_context() if tls: context.set_endpoint("docker", host=host, tls_cfg=tls, skip_tls_verify=not verify) if not context.is_docker_host(): raise UserError( "The platform targeted with the current context is not supported.\n" "Make sure the context in use targets a Docker Engine.\n") kwargs['base_url'] = context.Host if context.TLSConfig: kwargs['tls'] = context.TLSConfig if version: kwargs['version'] = version timeout = environment.get('COMPOSE_HTTP_TIMEOUT') if timeout: kwargs['timeout'] = int(timeout) else: kwargs['timeout'] = HTTP_TIMEOUT kwargs['user_agent'] = generate_user_agent() # Workaround for # https://pyinstaller.readthedocs.io/en/v3.3.1/runtime-information.html#ld-library-path-libpath-considerations if 'LD_LIBRARY_PATH_ORIG' in environment: kwargs['credstore_env'] = { 'LD_LIBRARY_PATH': environment.get('LD_LIBRARY_PATH_ORIG'), } client = APIClient(**kwargs) client._original_base_url = kwargs.get('base_url') return client
def __init__(self, build_job, repo_path, from_image, copy_code=True, build_steps=None, env_vars=None, dockerfile_name='Dockerfile'): self.build_job = build_job self.job_uuid = build_job.uuid.hex self.job_name = build_job.unique_name self.from_image = from_image self.image_name = get_image_name(self.build_job) self.image_tag = self.job_uuid self.folder_name = repo_path.split('/')[-1] self.repo_path = repo_path self.copy_code = copy_code self.build_path = '/'.join(self.repo_path.split('/')[:-1]) self.build_steps = get_list(build_steps) self.env_vars = get_list(env_vars) self.dockerfile_path = os.path.join(self.build_path, dockerfile_name) self.polyaxon_requirements_path = self._get_requirements_path() self.polyaxon_setup_path = self._get_setup_path() self.docker = APIClient(version='auto') self.registry_host = None self.docker_url = None
def __init__( self, info, dockerfile, repository, tag, to_build, context=None, ): # Meta-data about the image should go to info. # All keys in info are accessible as attributes # of this class self.info = info self.summary = {} self.dockerfile = dockerfile self.context = context # TODO: Add ability to tag image with multiple tags self.repository = repository self.tag = tag self.ecr_url = f"{self.repository}:{self.tag}" if not isinstance(to_build, bool): to_build = True if to_build == "true" else False self.to_build = to_build self.build_status = None self.client = APIClient(base_url=constants.DOCKER_URL) self.log = []