def _check_image(self):
if self._new_charts_image is None:
return False
address = self._new_charts_image.split("/")
if len(address) != 3:
logger.error("The format of the image %s can not be converted" %
self._new_charts_image)
return False
try:
ip, namespace, repository, tag = address[0], address[1], address[
2].split(":")[0], address[2].split(":")[1]
except Exception as e:
logger.error("The format of the image %s can not be converted" %
self._new_charts_image)
return False
try:
client = DockerRegistryClient("https://" + ip, verify_ssl=False)
charts_repo = client.repository(repository, namespace)
charts_tags = charts_repo.tags()
if tag in charts_tags:
logger.info("Find image: " + self._new_charts_image)
return True
else:
logger.error("Could not find image: " + self._new_charts_image)
return False
except Exception as e:
logger.error("Image query failed: {}".format(
self._new_charts_image))
return False
def get_tags(repository):
if '/' in sys.argv[1]:
registry, repository = repository.split('/', 1)
else:
registry, repository = \
'docker.io', 'library/{}'.format(repository)
if 'docker.io' in registry:
# XXX: more magic hackery
from six.moves.urllib.parse import urlsplit
REGISTRY_SERVICE_URL = "https://registry.docker.io"
HOST_SERVICE_URL = "https://registry-1.docker.io"
AUTH_SERVICE_URL = "https://auth.docker.io"
client = DockerRegistryClient(HOST_SERVICE_URL,
api_version=2,
auth_service_url=AUTH_SERVICE_URL)
client._base_client.auth.registry = urlsplit(
REGISTRY_SERVICE_URL).netloc
else:
client = DockerRegistryClient("https://{}".format(registry),
api_version=2)
repo = client.repository(repository)
tags = repo.tags()
# sort
tags = ['.'.join(tag) for tag in sorted([tag.split('.') for tag in tags])]
return tags
def scan_registry(registry_url):
"""Scan the docker registry and import the layers into Neo4J."""
client = DockerRegistryClient(registry_url)
try:
repositories = client.repositories()
except requests.HTTPError as e:
if e.response.status_code == requests.codes.not_found:
print("Catalog/Search not supported")
else:
raise
else:
print("Repositories:")
for repository in repositories:
repo = client.repository(repository)
for tag in repo.tags():
print("%s/%s:%s" % (registry_url, repository, tag))
assert client.api_version in [1, 2]
if client.api_version == 2:
manifest, digest = repo.manifest(tag)
layers = list(map(get_hash, manifest["fsLayers"]))
else:
image = repo.image(tag)
image_json = image.get_json()
layers = list(map(get_hash, image_json["fsLayers"]))
layer_fingerprint = "".join(layers)
with neo4j.session() as session:
session.run(
"MERGE ( i:Image {url: '%s', repo: '%s', tag: '%s'}) SET i.fingerprint='%s' "
% (registry_url, repository, tag, layer_fingerprint))
def test_repositories(self, version, namespace):
url = mock_registry(version)
client = DockerRegistryClient(url)
repositories = client.repositories(TEST_NAMESPACE)
assert len(repositories) == 1
assert TEST_NAME in repositories
repository = repositories[TEST_NAME]
assert repository.name == "%s/%s" % (TEST_NAMESPACE, TEST_REPO)
def test_repositories(self, version, namespace):
url = mock_registry(version)
client = DockerRegistryClient(url)
repositories = client.repositories(TEST_NAMESPACE)
assert len(repositories) == 1
assert TEST_NAME in repositories
repository = repositories[TEST_NAME]
assert repository.name == "%s/%s" % (TEST_NAMESPACE, TEST_REPO)
def test_repository_tags(self, version):
url = mock_registry(version)
client = DockerRegistryClient(url)
repositories = client.repositories(TEST_NAMESPACE)
assert TEST_NAME in repositories
repository = repositories[TEST_NAME]
tags = repository.tags()
assert len(tags) == 1
assert TEST_TAG in tags
def test_repository_tags(self, version):
url = mock_registry(version)
client = DockerRegistryClient(url)
repositories = client.repositories(TEST_NAMESPACE)
assert TEST_NAME in repositories
repository = repositories[TEST_NAME]
tags = repository.tags()
assert len(tags) == 1
assert TEST_TAG in tags
def test_api_version(self, client_api_version, registry_api_version,
should_succeed):
url = mock_registry(registry_api_version)
if should_succeed:
client = DockerRegistryClient(url, api_version=client_api_version)
assert client.api_version == client_api_version
else:
with pytest.raises(HTTPError):
client = DockerRegistryClient(url,
api_version=client_api_version)
client.refresh()
def get_docker_image_labels(image):
"""
Get labels from docker image
:param image: docker image
:type image: str
:return: dict[str, Any] -- image labels
"""
LOGGER.info('Getting labels for {} image'.format(image))
image_attributes = parse_docker_image_url(image)
# Get nexus registry host from ENV or image url
try:
if image_attributes.host == os.getenv('MODEL_IMAGES_REGISTRY_HOST'):
registry_host = os.getenv(legion.config.NEXUS_DOCKER_REGISTRY[0])
else:
if urllib3.util.parse_url(image_attributes.host).port == 443:
registry_host = 'https://{}'.format(image_attributes.host)
else:
registry_host = 'http://{}'.format(image_attributes.host)
except Exception as err:
LOGGER.error(
'Can\'t get registry host neither from ENV nor from image URL: {}'.
format(err))
raise err
try:
registry_client = DockerRegistryClient(
host=registry_host,
username=os.getenv(*legion.config.DOCKER_REGISTRY_USER),
password=os.getenv(*legion.config.DOCKER_REGISTRY_PASSWORD),
api_version=2)
manifest = registry_client.repository(image_attributes.repo).manifest(
image_attributes.ref)
labels = json.loads(manifest[0]["history"][0]
["v1Compatibility"])["container_config"]["Labels"]
except Exception as err:
raise Exception('Can\'t get image labels for {} image: {}'.format(
image, err))
required_headers = [
legion.containers.headers.DOMAIN_MODEL_ID,
legion.containers.headers.DOMAIN_MODEL_VERSION,
legion.containers.headers.DOMAIN_CONTAINER_TYPE
]
if any(header not in labels for header in required_headers):
raise Exception(
'Missed one of %s labels. Available labels: %s' %
(', '.join(required_headers), ', '.join(tuple(labels.keys()))))
return labels
def run(self):
args = self.parser.parse_args()
basic_config_args = {}
if args.verbose:
basic_config_args['level'] = logging.DEBUG
elif args.quiet:
basic_config_args['level'] = logging.WARNING
logging.basicConfig(**basic_config_args)
kwargs = {
'username': args.username,
'password': args.password,
}
if args.api_version:
kwargs['api_version'] = args.api_version
client = DockerRegistryClient(
args.registry[0],
auth_service_url=args.authorization_service,
verify_ssl=args.verify_ssl,
**kwargs)
if args.repository:
if args.ref:
self.show_manifest(client, args.repository, args.ref)
else:
self.show_tags(client, args.repository)
else:
self.show_repositories(client)
def __init__(self, **kwargs):
self.address = kwargs.get('address', 'http://127.0.0.1:5000/')
self.verify_ssl = kwargs.get('verify_ssl', True)
self.api_version = kwargs.get('api_version', 2)
self.credentials = kwargs.get('credentials', '')
self._client = DockerRegistryClient(self.address,
verify_ssl=self.verify_ssl,
api_version=self.api_version)
def get_client(registry, user, pw):
# auth_service_url_full = f'{registry}/v2/token'
cl = DockerRegistryClient(
host=registry,
api_version=1,
username=user,
password=pw,
)
# cl._base_client.check_status()
return cl
def get_tags(self, image):
params = {}
for (k, v) in self.config.items():
if v == '' or v is None:
continue
# FOR AWS ECR User
if k == 'username' and v == 'AWS':
# get password
region_name = self._get_aws_region_from_host(self.config['host'])
password = self._get_aws_ecr_token(region_name)
if password:
params.update({'password': password})
params.update({k: v})
try:
client = DockerRegistryClient(**params)
r = client.repository(image)
tags = r.tags()
# return sorted list
return tags
except Exception as e:
# Hard to determine backend problems
_LOGGER.error(f"Error to get container tags: {e}")
raise ERROR_REPOSITORY_BACKEND(host=params['host'])
def test_api_version(self, client_api_version, registry_api_version,
should_succeed):
url = mock_registry(registry_api_version)
if should_succeed:
client = DockerRegistryClient(url, api_version=client_api_version)
assert client.api_version == client_api_version
else:
with pytest.raises(HTTPError):
client = DockerRegistryClient(url,
api_version=client_api_version)
client.refresh()
src_registry_url = config['source_registry']['url']
dst_registry_url = config['destination_registry']['url']
src_username = None if 'username' not in config[
'source_registry'] else str(config['source_registry']['username'])
src_password = determine_password(config, 'src')
dst_username = None if 'username' not in config[
'destination_registry'] else str(
config['destination_registry']['username'])
dst_password = determine_password(config, 'dst')
src_client = DockerRegistryClient(src_registry_url,
username=src_username,
password=src_password,
verify_ssl=False)
dst_client = DockerRegistryClient(dst_registry_url,
username=dst_username,
password=dst_password,
verify_ssl=False)
docker_client = docker.from_env()
docker_client.login(registry=src_registry_url,
username=src_username,
password=src_password)
docker_client.login(registry=dst_registry_url,
username=dst_username,
password=dst_password)
src_registry_url = config['source_registry']['url']
dst_registry_url = config['destination_registry']['url']
src_username = None if 'username' not in config[
'source_registry'] else str(config['source_registry']['username'])
src_password = determine_password(config, 'src')
dst_username = None if 'username' not in config[
'destination_registry'] else str(
config['destination_registry']['username'])
dst_password = determine_password(config, 'dst')
src_client = DockerRegistryClient(src_registry_url,
username=src_username,
password=src_password)
dst_client = DockerRegistryClient(dst_registry_url,
username=dst_username,
password=dst_password)
docker_client = docker.from_env()
docker_client.login(registry=src_registry_url,
username=src_username,
password=src_password)
docker_client.login(registry=dst_registry_url,
username=dst_username,
password=dst_password)
from docker_registry_client import DockerRegistryClient
import docker
client = DockerRegistryClient("http://127.0.0.1:5000")
print client.repositories()
r = client.repository("busybox")
tags = r.tags()
print tags
docker_client = docker.from_env()
reg = docker_client.containers.get("registry")
print reg
docker_client.login(registry="http://127.0.0.1:5000", username="", password="")
docker_client.images.pull("127.0.0.1:5000/busybox")
def test_repository_manifest(self):
url = mock_v2_registry()
client = DockerRegistryClient(url)
repository = client.repositories()[TEST_NAME]
manifest, digest = repository.manifest(TEST_TAG)
repository.delete_manifest(digest)
import os
from laceworksdk import LaceworkClient
from docker_registry_client import DockerRegistryClient
lw = LaceworkClient(account=os.getenv('LW_ACCOUNT'),
api_key=os.getenv('LW_API_KEY'),
api_secret=os.getenv('LW_API_SECRET'))
registry = os.getenv('REGISTRY')
nexus = DockerRegistryClient(f"https://{registry}",
verify_ssl=False,
username=os.getenv('REGISTRY_USER'),
password=os.getenv('REGISTRY_PASSWORD'))
repos = nexus.repositories()
for name, repo in repos.items():
tags = repo.tags()
for tag in tags:
scan_request = lw.vulnerabilities.initiate_container_scan(
registry, name, tag)
print(
f"INITIATING SCAN FOR -> REGISTRY[{registry}] IMAGE[{name}] TAG[{tag}] -> RequestId [{scan_request['data']['RequestId']}]"
)
def test_repository_namespace_incorrect(self, version):
url = mock_registry(version)
client = DockerRegistryClient(url)
with pytest.raises(RuntimeError):
client.repository('{0}/{1}'.format(TEST_NAMESPACE, TEST_REPO),
namespace=TEST_NAMESPACE)
def test_repository_namespace_incorrect(self, version):
url = mock_registry(version)
client = DockerRegistryClient(url)
with pytest.raises(RuntimeError):
client.repository('{0}/{1}'.format(TEST_NAMESPACE, TEST_REPO),
namespace=TEST_NAMESPACE)
def test_repository(self, version, repository, namespace):
url = mock_registry(version)
client = DockerRegistryClient(url)
repository = client.repository(repository, namespace=namespace)
assert isinstance(repository, BaseRepository)
def test_namespaces(self, version):
url = mock_registry(version)
client = DockerRegistryClient(url)
assert client.namespaces() == [TEST_NAMESPACE]
def test_api_version_in_use(self, version):
url = mock_registry(version)
client = DockerRegistryClient(url)
assert client.api_version == version
def test_namespaces(self, version):
url = mock_registry(version)
client = DockerRegistryClient(url)
assert client.namespaces() == [TEST_NAMESPACE]
def test_repository(self, version, repository, namespace):
url = mock_registry(version)
client = DockerRegistryClient(url)
repository = client.repository(repository, namespace=namespace)
assert isinstance(repository, BaseRepository)
import docker
from requests.exceptions import HTTPError
import yaml
import os
import io
import tarfile
from docker_registry_client import DockerRegistryClient
target_directory = 'sample_lain_yaml'
try:
os.mkdir(target_directory)
except FileExistsError:
pass
client = docker.from_env(version='auto')
registry_host = 'registry.lain.ein.plus'
registry = DockerRegistryClient(f'http://{registry_host}')
repos = registry.repositories()
print(repos)
for repo in repos.values():
try:
tags = repo.tags()
except HTTPError:
continue
if not tags:
continue
try:
latest_meta_tag = max(t for t in tags if t.startswith('meta-'))
except ValueError:
continue
image_name = f'{registry_host}/{repo.name}:{latest_meta_tag}'
def test_repository_manifest(self):
url = mock_v2_registry()
client = DockerRegistryClient(url)
repository = client.repositories()[TEST_NAME]
manifest, digest = repository.manifest(TEST_TAG)
repository.delete_manifest(digest)